Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

onedrive: authenticate SharePoint downloads, uploads and copies after tempauth retirement

Nick Craig-Wood84bdebae77aa
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index eb253c7eb..5524dc0d4 100644
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -41,6 +41,7 @@ import (
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
+ "golang.org/x/oauth2"
)
const (
@@ -844,19 +845,33 @@ type Options struct {
// Fs represents a remote OneDrive
type Fs struct {
- name string // name of this remote
- root string // the path we are working on
- opt Options // parsed options
- ci *fs.ConfigInfo // global config
- features *fs.Features // optional features
- srv *rest.Client // the connection to the OneDrive server
- unAuth *rest.Client // no authentication connection to the OneDrive server
- dirCache *dircache.DirCache // Map of directory path to directory id
- pacer *fs.Pacer // pacer for API calls
- tokenRenewer *oauthutil.Renew // renew the token on expiry
- driveID string // ID to use for querying Microsoft Graph
- driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
- hashType hash.Type // type of the hash we are using
+ name string // name of this remote
+ root string // the path we are working on
+ opt Options // parsed options
+ ci *fs.ConfigInfo // global config
+ features *fs.Features // optional features
+ srv *rest.Client // the connection to the OneDrive server
+ unAuth *rest.Client // no authentication connection to the OneDrive server
+ dirCache *dircache.DirCache // Map of directory path to directory id
+ pacer *fs.Pacer // pacer for API calls
+ tokenRenewer *oauthutil.Renew // renew the token on expiry
+ tokenSource *oauthutil.TokenSource // source of the API token
+ sharePoint *sharePointClients // authenticated clients for SharePoint hosts
+ driveID string // ID to use for querying Microsoft Graph
+ driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
+ hashType hash.Type // type of the hash we are using
+}
+
+// sharePointClients holds the clients authenticated for SharePoint hosts
+type sharePointClients struct {
+ mu sync.Mutex
+ hosts map[string]sharePointAuth
+}
+
+// sharePointAuth is a client authenticated for a SharePoint host
+type sharePointAuth struct {
+ client *rest.Client
+ tokenSource oauth2.TokenSource
}
// Object describes a OneDrive object
@@ -1076,6 +1091,154 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
+// errPreauthRetired explains an authentication failure on a URL which
+// SharePoint used to return pre-authenticated.
+var errPreauthRetired = errors.New("SharePoint refused to authenticate a request - Microsoft is retiring pre-authenticated (tempauth) URLs, see https://rclone.org/onedrive/#pre-authenticated-urls")
+
+// sharePointDomains are the domains of SharePoint Online in the
+// different Microsoft clouds
+var sharePointDomains = []string{".sharepoint.com", ".sharepoint.us", ".sharepoint-mil.us", ".sharepoint.cn", ".sharepoint.de"}
+
+// stripTempauth removes the tempauth parameter from the URLs the API
+// returns, which simulates the retirement of pre-authenticated URLs
+// in tests.
+var stripTempauth = false
+
+// returnedURL processes a URL returned by the API before use
+func returnedURL(rawURL string) string {
+ if !stripTempauth {
+ return rawURL
+ }
+ u, err := url.Parse(rawURL)
+ if err != nil {
+ return rawURL
+ }
+ q := u.Query()
+ q.Del("tempauth")
+ u.RawQuery = q.Encode()
+ return u.String()
+}
+
+// urlAuth says how a URL returned by the API is authenticated
+type urlAuth int
+
+const (
+ urlAuthNone urlAuth = iota // call without authentication
+ urlAuthAPI // call with the API token
+ urlAuthSharePoint // call with a token for the SharePoint host
+)
+
+// apiHost returns the host the API calls made with f.srv go to.
+func (f *Fs) apiHost() string {
+ endpoint := graphAPIEndpoint[f.opt.Region]
+ if f.opt.TenantURL != "" {
+ endpoint = f.opt.TenantURL
+ }
+ u, err := url.Parse(endpoint)
+ if err != nil || u.Scheme != "https" {
+ return ""
+ }
+ return u.Host
+}
+
+// urlAuthFor returns how to authenticate a URL returned by the API,
+// such as an upload session or an async job monitor.
+//
+// Pre-authenticated URLs fail if they are sent an Authorization
+// header and tokens must not be sent to other hosts, so only HTTPS
+// URLs without tempauth on the API host or a SharePoint host are
+// authenticated.
+func (f *Fs) urlAuthFor(rawURL string) urlAuth {
+ u, err := url.Parse(rawURL)
+ if err != nil || u.Scheme != "https" || u.Query().Has("tempauth") {
+ return urlAuthNone
+ }
+ if apiHost := f.apiHost(); apiHost != "" && strings.EqualFold(u.Host, apiHost) {
+ return urlAuthAPI
+ }
+ if isSharePointHost(u.Hostname()) {
+ return urlAuthSharePoint
+ }
+ return urlAuthNone
+}
+
+// isSharePointHost returns true if host is in a SharePoint Online domain.
+func isSharePointHost(host string) bool {
+ host = strings.ToLower(host)
+ for _, domain := range sharePointDomains {
+ if strings.HasSuffix(host, domain) {
+ return true
+ }
+ }
+ return false
+}
+
+// checkSharePointRedirect stops the SharePoint token being sent to a
+// host which isn't SharePoint, as the token is added to every request
+// the client makes.
+func checkSharePointRedirect(req *http.Request, via []*http.Request) error {
+ if len(via) >= 10 {
+ return errors.New("stopped after 10 redirects")
+ }
+ if req.URL.Scheme != "https" || !isSharePointHost(req.URL.Hostname()) {
+ return fmt.Errorf("refusing to send a SharePoint token on a redirect to %s://%s", req.URL.Scheme, req.URL.Host)
+ }
+ return nil
+}
+
+// clientForURL returns the client to call a URL returned by the API with.
+func (f *Fs) clientForURL(ctx context.Context, rawURL string) (*rest.Client, error) {
+ switch f.urlAuthFor(rawURL) {
+ case urlAuthAPI:
+ return f.srv, nil
+ case urlAuthSharePoint:
+ u, _ := url.Parse(rawURL)
+ return f.sharePointClient(ctx, u.Host)
+ }
+ return f.unAuth, nil
+}
+
+// sharePointClient returns a client authenticated for the SharePoint
+// host, which upload session and async job monitor URLs need once
+// they are no longer pre-authenticated.
+//
+// The token is got from the existing credentials, which Entra ID
+// allows for SharePoint as the user has consented to the Graph file
+// scopes.
+func (f *Fs) sharePointClient(ctx context.Context, host string) (*rest.Client, error) {
+ if f.tokenSource == nil || f.sharePoint == nil {
+ return nil, fmt.Errorf("%w: no credentials to get a SharePoint token with", errPreauthRetired)
+ }
+ host = strings.ToLower(host)
+ f.sharePoint.mu.Lock()
+ defer f.sharePoint.mu.Unlock()
+ sp, ok := f.sharePoint.hosts[host]
+ if !ok {
+ tokenSource := f.tokenSource.ScopedTokenSource([]string{"https://" + host + "/.default"})
+ client := oauth2.NewClient(oauthutil.Context(ctx, fshttp.NewClient(ctx)), tokenSource)
+ client.CheckRedirect = checkSharePointRedirect
+ sp = sharePointAuth{client: rest.NewClient(client), tokenSource: tokenSource}
+ f.sharePoint.hosts[host] = sp
+ fs.Debugf(f, "Using a SharePoint token for %s", host)
+ }
+ // Get the token now so failures are reported before the client is used
+ if _, err := sp.tokenSource.Token(); err != nil {
+ return nil, fmt.Errorf("%w: failed to get a SharePoint token: %w", errPreauthRetired, err)
+ }
+ return sp.client, nil
+}
+
+// checkPreauthRetired adds an explanation to a 401 error from a URL
+// returned by the API which needs a SharePoint token, as these were
+// pre-authenticated until Microsoft retired them.
+func (f *Fs) checkPreauthRetired(rawURL string, resp *http.Response, err error) error {
+ if err == nil || resp == nil || resp.StatusCode != http.StatusUnauthorized ||
+ f.driveType == driveTypePersonal || errors.Is(err, errPreauthRetired) || f.urlAuthFor(rawURL) != urlAuthSharePoint {
+ return err
+ }
+ return fmt.Errorf("%w: %w", errPreauthRetired, err)
+}
+
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.SizeSuffixBase
if cs%chunkSizeMultiple != 0 {
@@ -1144,16 +1307,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci := fs.GetConfig(ctx)
f := &Fs{
- name: name,
- root: root,
- opt: *opt,
- ci: ci,
- driveID: opt.DriveID,
- driveType: opt.DriveType,
- srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
- unAuth: rest.NewClient(client).SetRoot(rootURL),
- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
- hashType: QuickXorHashType,
+ name: name,
+ root: root,
+ opt: *opt,
+ ci: ci,
+ driveID: opt.DriveID,
+ driveType: opt.DriveType,
+ srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
+ tokenSource: ts,
+ sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)},
+ unAuth: rest.NewClient(client).SetRoot(rootURL),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ hashType: QuickXorHashType,
}
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -1725,17 +1890,25 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
var resp *http.Response
var err error
var body []byte
+ var redirected bool
err = f.pacer.Call(func() (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
if err != nil {
return false, err
}
- // The monitor URL needs no authentication, but it should still
- // go through the configured transport and honour ctx
- resp, err = f.unAuth.Do(req)
+ client, err := f.clientForURL(ctx, location)
+ if err != nil {
+ return false, err
+ }
+ resp, err = client.Do(req)
if err != nil {
return fserrors.ShouldRetry(err), err
}
+ redirected = resp.Request.URL.String() != req.URL.String()
+ if resp.StatusCode == http.StatusUnauthorized {
+ _ = resp.Body.Close()
+ return false, f.checkPreauthRetired(location, resp, fmt.Errorf("async job monitor: HTTP error %s", resp.Status))
+ }
body, err = rest.ReadBody(resp)
return fserrors.ShouldRetry(err), err
})
@@ -1748,6 +1921,11 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
if err != nil {
return fmt.Errorf("async status result not JSON: %q: %w", body, err)
}
+ // An authenticated monitor redirects to the new item when the
+ // job has finished instead of returning its status
+ if status.Status == "" && redirected {
+ status.Status = "completed"
+ }
switch status.Status {
case "failed":
@@ -1847,7 +2025,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
}
// read location header
- location := resp.Header.Get("Location")
+ location := returnedURL(resp.Header.Get("Location"))
if location == "" {
return nil, errors.New("didn't receive location header in copy response")
}
@@ -1858,6 +2036,10 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
fs.Debugf(src, "Server-side copy failed - file not shared between drives")
return nil, fs.ErrorCantCopy
}
+ if errors.Is(err, errPreauthRetired) {
+ fs.Debugf(src, "Server-side copy can't be monitored - copying another way: %v", err)
+ return nil, fs.ErrorCantCopy
+ }
if err != nil {
return nil, err
}
@@ -2532,16 +2714,23 @@ func (o *Object) openContentPrefer(ctx context.Context, options ...fs.OpenOption
return o.openWithRedirect(ctx, &opts)
}
-// openWithRedirect downloads via /content style endpoints that 302 to a preauthenticated URL.
+// openWithRedirect downloads via /content style endpoints that 302 to
+// a download URL, which is pre-authenticated or needs a SharePoint
+// token.
func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.ReadCloser, err error) {
var resp *http.Response
- // Make a note of the redirect target as we need to call it without Auth
+ // Make a note of the redirect target as it needs different auth
var redirectReq *http.Request
opts.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
- // Preauthenticated download URLs must not carry the Graph Authorization header.
+ if stripTempauth {
+ if u, err := url.Parse(returnedURL(req.URL.String())); err == nil {
+ req.URL = u
+ }
+ }
+ // Download URLs must not carry the Graph Authorization header.
req.Header.Del("Authorization")
// Keep Prefer on the redirect when forcing infected download; some SharePoint
// endpoints honor it only on the final download request.
@@ -2580,17 +2769,24 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
return nil, err
}
if redirectReq != nil {
+ redirectURL := redirectReq.URL.String()
+ var client *rest.Client
+ client, err = o.fs.clientForURL(ctx, redirectURL)
+ if err != nil {
+ return nil, err
+ }
err = o.fs.pacer.Call(func() (bool, error) {
- resp, err = o.fs.unAuth.Do(redirectReq)
+ resp, err = client.Do(redirectReq)
if err != nil {
return shouldRetry(ctx, resp, err)
}
- // unAuth.Do does not check status; treat non-2xx as failure so a malware
+ // Do does not check status; treat non-2xx as failure so a malware
// JSON body is not written out as file content.
if resp.StatusCode < 200 || resp.StatusCode > 299 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
_ = resp.Body.Close()
err = fmt.Errorf("HTTP error %d (%s) %s", resp.StatusCode, resp.Status, strings.TrimSpace(string(body)))
+ err = o.fs.checkPreauthRetired(redirectURL, resp, err)
if strings.Contains(string(body), "malwareDetected") || resp.Header.Get("X-Virus-Infected") != "" {
err = malwareDownloadError(o.fs.opt.AVOverride, err)
}
@@ -2652,12 +2848,16 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
}
var info api.UploadFragmentResponse
var resp *http.Response
+ client, err := o.fs.clientForURL(ctx, url)
+ if err != nil {
+ return 0, err
+ }
err = o.fs.pacer.Call(func() (bool, error) {
- resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
+ resp, err = client.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(ctx, resp, err)
})
if err != nil {
- return 0, err
+ return 0, o.fs.checkPreauthRetired(url, resp, err)
}
if len(info.NextExpectedRanges) != 1 {
return 0, fmt.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)
@@ -2681,6 +2881,10 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
var resp *http.Response
var body []byte
skip := int64(0)
+ client, err := o.fs.clientForURL(ctx, url)
+ if err != nil {
+ return nil, err
+ }
err = o.fs.pacer.Call(func() (bool, error) {
toSend := chunkSize - skip
opts := rest.Opts{
@@ -2692,7 +2896,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
Options: options,
}
_, _ = chunk.Seek(skip, io.SeekStart)
- resp, err = o.fs.unAuth.Call(ctx, &opts)
+ resp, err = client.Call(ctx, &opts)
if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
pos, posErr := o.getPosition(ctx, url)
@@ -2731,7 +2935,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
}
return false, nil
})
- return info, err
+ return info, o.fs.checkPreauthRetired(url, resp, err)
}
// cancelUploadSession cancels an upload session
@@ -2742,11 +2946,15 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
NoResponse: true,
}
var resp *http.Response
+ client, err := o.fs.clientForURL(ctx, url)
+ if err != nil {
+ return err
+ }
err = o.fs.pacer.Call(func() (bool, error) {
- resp, err = o.fs.unAuth.Call(ctx, &opts)
+ resp, err = client.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
- return
+ return o.fs.checkPreauthRetired(url, resp, err)
}
// uploadMultipart uploads a file using multipart upload
@@ -2764,7 +2972,20 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
if err != nil {
return nil, err
}
- uploadURL := session.UploadURL
+ uploadURL := returnedURL(session.UploadURL)
+
+ // Check a session which needs SharePoint authentication can get it
+ // before reading any data, so it can be uploaded in a single part
+ // instead if it can't.
+ if o.fs.urlAuthFor(uploadURL) == urlAuthSharePoint {
+ if _, err = o.getPosition(ctx, uploadURL); err != nil {
+ if !errors.Is(err, errPreauthRetired) || size >= int64(maxSinglePartSize) {
+ return nil, err
+ }
+ fs.Debugf(o, "Uploading in a single part as the upload session can't be used: %v", err)
+ return o.uploadSinglepart(ctx, in, src, options...)
+ }
+ }
// Cancel the session if something went wrong
defer atexit.OnError(&err, func() {
diff --git a/backend/onedrive/onedrive_internal_test.go b/backend/onedrive/onedrive_internal_test.go
index de576c531..397a7ef51 100644
--- a/backend/onedrive/onedrive_internal_test.go
+++ b/backend/onedrive/onedrive_internal_test.go
@@ -3,6 +3,7 @@ package onedrive
import (
"context"
"encoding/json"
+ "flag"
"fmt"
"net/http"
"net/http/httptest"
@@ -44,6 +45,10 @@ const (
// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_invite?view=odsp-graph-online#http-request-1
)
+func init() {
+ flag.BoolVar(&stripTempauth, "strip-tempauth", false, "Remove tempauth from URLs the API returns to simulate the retirement of pre-authenticated URLs")
+}
+
// TestMain drives the tests
func TestMain(m *testing.M) {
fstest.TestMain(m)
@@ -818,3 +823,49 @@ func TestWaitForJobCancel(t *testing.T) {
t.Fatal("waitForJob did not return after the context was cancelled")
}
}
+
+// TestWaitForJobUnauthorized checks that a monitor URL which refuses
+// the request fails straight away instead of polling until the timeout.
+func TestWaitForJobUnauthorized(t *testing.T) {
+ var polls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ polls.Add(1)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusUnauthorized)
+ _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "unauthenticated", "message": "no token"}})
+ }))
+ t.Cleanup(server.Close)
+ f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+ f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+ f.driveType = driveTypeBusiness
+
+ err := f.waitForJob(context.Background(), server.URL+"/monitor", &Object{remote: "file.txt"})
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "401")
+ assert.Equal(t, int32(1), polls.Load())
+}
+
+// TestWaitForJobRedirectCompleted checks that a monitor which
+// redirects to the new item, as it does when authenticated, is taken
+// to mean the job has finished.
+func TestWaitForJobRedirectCompleted(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/monitor":
+ http.Redirect(w, r, "/item", http.StatusSeeOther)
+ case "/item":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = fmt.Fprint(w, `{"id":"newid","name":"file.txt","size":5,"file":{}}`)
+ default:
+ t.Errorf("unexpected request %s", r.URL)
+ }
+ }))
+ t.Cleanup(server.Close)
+ f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+ f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt", hasMetaData: true})
+ require.NoError(t, err)
+}
diff --git a/backend/onedrive/upload_internal_test.go b/backend/onedrive/upload_internal_test.go
index aae575466..0150e62e7 100644
--- a/backend/onedrive/upload_internal_test.go
+++ b/backend/onedrive/upload_internal_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
@@ -13,11 +14,14 @@ import (
"time"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "golang.org/x/oauth2"
)
// fragmentServer is a fake OneDrive upload session which records the
@@ -66,6 +70,8 @@ func (s *fragmentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusAccepted)
_, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)
+ case r.Method == "DELETE" && r.URL.Path == "/upload":
+ w.WriteHeader(http.StatusNoContent)
default:
s.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
@@ -118,3 +124,298 @@ func TestUploadMultipartSkip(t *testing.T) {
assert.Equal(t, 2, s.fragments)
assert.Equal(t, 3, s.attempts, "expected one 416, a partial re-send and the final fragment")
}
+
+// TestURLAuthFor checks that tokens are only sent to HTTPS URLs on
+// the API host or a SharePoint host which aren't pre-authenticated.
+func TestURLAuthFor(t *testing.T) {
+ const tenant = "https://contoso-my.sharepoint.com/_api"
+ for _, tc := range []struct {
+ name string
+ region string
+ tenantURL string
+ url string
+ want urlAuth
+ }{
+ {"graph", "global", "", "https://graph.microsoft.com/v1.0/drives/x/items/y/uploadSession", urlAuthAPI},
+ {"graph case", "global", "", "https://Graph.Microsoft.com/v1.0/monitor/abc", urlAuthAPI},
+ {"graph tempauth", "global", "", "https://graph.microsoft.com/v1.0/upload?tempauth=abc", urlAuthNone},
+ {"graph http", "global", "", "http://graph.microsoft.com/v1.0/upload", urlAuthNone},
+ {"graph other region", "global", "", "https://graph.microsoft.us/v1.0/upload", urlAuthNone},
+ {"graph us", "us", "", "https://graph.microsoft.us/v1.0/upload", urlAuthAPI},
+ {"graph lookalike", "global", "", "https://graph.microsoft.com.example.com/upload", urlAuthNone},
+ {"sharepoint tempauth", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x&tempauth=abc", urlAuthNone},
+ {"sharepoint", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x", urlAuthSharePoint},
+ {"sharepoint case", "global", "", "https://Contoso.SharePoint.com/_api/v2.0/monitor/abc", urlAuthSharePoint},
+ {"sharepoint us", "us", "", "https://contoso.sharepoint.us/_api/v2.0/upload", urlAuthSharePoint},
+ {"sharepoint cn", "cn", "", "https://contoso.sharepoint.cn/_api/v2.0/upload", urlAuthSharePoint},
+ {"sharepoint http", "global", "", "http://contoso-my.sharepoint.com/_api/v2.0/upload", urlAuthNone},
+ {"sharepoint lookalike", "global", "", "https://contoso.sharepoint.com.example.com/upload", urlAuthNone},
+ {"sharepoint suffix only", "global", "", "https://evilsharepoint.com/upload", urlAuthNone},
+ {"personal", "global", "", "https://api.onedrive.com/rup/abc", urlAuthNone},
+ {"personal upload", "global", "", "https://my.microsoftpersonalcontent.com/personal/abc/_api/v2.0/drive/items/x/uploadSession", urlAuthNone},
+ {"unparseable", "global", "", "https://graph.microsoft.com/%zz", urlAuthNone},
+ {"tenant", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/monitor/abc", urlAuthAPI},
+ {"tenant tempauth", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/upload?tempauth=abc", urlAuthNone},
+ {"tenant graph", "global", tenant, "https://graph.microsoft.com/v1.0/upload", urlAuthNone},
+ {"tenant other sharepoint", "global", tenant, "https://contoso.sharepoint.com/_api/v2.0/upload", urlAuthSharePoint},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := &Fs{}
+ f.opt.Region = tc.region
+ f.opt.TenantURL = tc.tenantURL
+ assert.Equal(t, tc.want, f.urlAuthFor(tc.url))
+ })
+ }
+}
+
+// TestCheckSharePointRedirect checks the SharePoint token is only
+// sent on redirects to SharePoint over HTTPS.
+func TestCheckSharePointRedirect(t *testing.T) {
+ for _, tc := range []struct {
+ url string
+ ok bool
+ }{
+ {"https://contoso-my.sharepoint.com/download.aspx", true},
+ {"https://205807-ipv4v6.gr.global.aa-rt.sharepoint.com/download.aspx?tempauth=x", true},
+ {"https://contoso.sharepoint.us/download.aspx", true},
+ {"http://contoso-my.sharepoint.com/download.aspx", false},
+ {"https://contoso.sharepoint.com.example.com/download.aspx", false},
+ {"https://graph.microsoft.com/v1.0/me", false},
+ {"https://example.com/", false},
+ } {
+ req, err := http.NewRequest("GET", tc.url, nil)
+ require.NoError(t, err)
+ err = checkSharePointRedirect(req, nil)
+ if tc.ok {
+ assert.NoError(t, err, tc.url)
+ } else {
+ assert.Error(t, err, tc.url)
+ }
+ }
+ req, err := http.NewRequest("GET", "https://contoso-my.sharepoint.com/", nil)
+ require.NoError(t, err)
+ assert.Error(t, checkSharePointRedirect(req, make([]*http.Request, 10)), "too many redirects")
+}
+
+// TestCheckPreauthRetired checks which 401 errors get the explanation.
+func TestCheckPreauthRetired(t *testing.T) {
+ const sharePointURL = "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x"
+ errBase := errors.New("HTTP error 401")
+ for _, tc := range []struct {
+ name string
+ driveType string
+ url string
+ status int
+ want bool
+ }{
+ {"sharepoint 401", driveTypeBusiness, sharePointURL, http.StatusUnauthorized, true},
+ {"library 401", driveTypeSharepoint, sharePointURL, http.StatusUnauthorized, true},
+ {"personal 401", driveTypePersonal, sharePointURL, http.StatusUnauthorized, false},
+ {"sharepoint 403", driveTypeBusiness, sharePointURL, http.StatusForbidden, false},
+ {"tempauth 401", driveTypeBusiness, sharePointURL + "&tempauth=abc", http.StatusUnauthorized, false},
+ {"graph 401", driveTypeBusiness, "https://graph.microsoft.com/v1.0/upload", http.StatusUnauthorized, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := &Fs{driveType: tc.driveType}
+ f.opt.Region = "global"
+ err := f.checkPreauthRetired(tc.url, &http.Response{StatusCode: tc.status}, errBase)
+ assert.ErrorIs(t, err, errBase)
+ assert.Equal(t, tc.want, errors.Is(err, errPreauthRetired), "err = %v", err)
+ })
+ }
+ f := &Fs{driveType: driveTypeBusiness}
+ assert.NoError(t, f.checkPreauthRetired(sharePointURL, &http.Response{StatusCode: http.StatusUnauthorized}, nil))
+ assert.ErrorIs(t, f.checkPreauthRetired(sharePointURL, nil, errBase), errBase)
+}
+
+// TestSharePointClient checks that the client for a SharePoint host
+// is authenticated with a token for that host got from the existing
+// refresh token, which is fetched once and not stored.
+func TestSharePointClient(t *testing.T) {
+ var scopes []string
+ var auths []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/token":
+ _ = r.ParseForm()
+ assert.Equal(t, "refresh_token", r.PostForm.Get("grant_type"))
+ assert.Equal(t, "refresh", r.PostForm.Get("refresh_token"))
+ scopes = append(scopes, r.PostForm.Get("scope"))
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = fmt.Fprint(w, `{"access_token":"sharepoint_token","token_type":"Bearer","expires_in":3600}`)
+ case "/upload":
+ auths = append(auths, r.Header.Get("Authorization"))
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = fmt.Fprint(w, `{"nextExpectedRanges":["0-"]}`)
+ case "/redirect":
+ http.Redirect(w, r, "/upload", http.StatusFound)
+ default:
+ t.Errorf("unexpected request %s %s", r.Method, r.URL)
+ }
+ }))
+ t.Cleanup(server.Close)
+
+ storedToken := `{"access_token":"graph_token","token_type":"Bearer","refresh_token":"refresh","expiry":"2099-01-01T00:00:00Z"}`
+ m := configmap.Simple{"token": storedToken}
+ ctx := context.Background()
+ _, ts, err := oauthutil.NewClientWithBaseClient(ctx, "remote", m, &oauthutil.Config{
+ ClientID: "client_id",
+ ClientSecret: "client_secret",
+ TokenURL: server.URL + "/token",
+ }, server.Client())
+ require.NoError(t, err)
+ f := &Fs{tokenSource: ts, sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)}}
+
+ for range 2 {
+ client, err := f.sharePointClient(ctx, "Contoso-my.sharepoint.com")
+ require.NoError(t, err)
+ _, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/upload"})
+ require.NoError(t, err)
+ }
+ assert.Equal(t, []string{"https://contoso-my.sharepoint.com/.default"}, scopes, "token should be fetched once for the host")
+ assert.Equal(t, []string{"Bearer sharepoint_token", "Bearer sharepoint_token"}, auths)
+ assert.Equal(t, storedToken, m["token"], "the stored token must not change")
+
+ // The token isn't sent on a redirect to a host which isn't SharePoint
+ client, err := f.sharePointClient(ctx, "contoso-my.sharepoint.com")
+ require.NoError(t, err)
+ _, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/redirect"})
+ assert.ErrorContains(t, err, "refusing to send a SharePoint token")
+ assert.Len(t, auths, 2)
+
+ // Without credentials there is no client and the error explains why
+ _, err = (&Fs{}).sharePointClient(ctx, "contoso-my.sharepoint.com")
+ assert.ErrorIs(t, err, errPreauthRetired)
+}
+
+// TestUploadSessionFallback checks that an upload session which needs
+// SharePoint authentication which can't be got is replaced by a single
+// part upload before any data is read.
+func TestUploadSessionFallback(t *testing.T) {
+ content := bytes.Repeat([]byte("onedrive"), 1024)
+ var requests []string
+ var uploaded []byte
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests = append(requests, r.Method+" "+r.URL.Path)
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case r.Method == "POST" && r.URL.Path == "/root/createUploadSession":
+ _, _ = fmt.Fprint(w, `{"uploadUrl":"https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x","nextExpectedRanges":["0-"]}`)
+ case r.Method == "PUT" && r.URL.Path == "/root/content":
+ var err error
+ uploaded, err = io.ReadAll(r.Body)
+ require.NoError(t, err)
+ _, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
+ case r.Method == "PATCH":
+ _, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
+ default:
+ t.Errorf("unexpected request %s %s", r.Method, r.URL)
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ t.Cleanup(server.Close)
+ client := rest.NewClient(server.Client()).SetRoot(server.URL)
+ f := &Fs{
+ ci: fs.GetConfig(context.Background()),
+ srv: client,
+ unAuth: client,
+ driveType: driveTypeBusiness,
+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+ }
+ f.opt.Region = "global"
+ f.opt.ChunkSize = fs.SizeSuffix(len(content) / 2)
+ o := &Object{fs: f}
+
+ src := object.NewStaticObjectInfo("remote", time.Now(), int64(len(content)), true, nil, nil)
+ _, err := o.uploadMultipart(context.Background(), bytes.NewReader(content), src)
+ require.NoError(t, err)
+ assert.Equal(t, content, uploaded)
+ assert.Equal(t, []string{"POST /root/createUploadSession", "PUT /root/content"}, requests[:2])
+}
+
+// redirectTransport sends every request to the test server, so URLs
+// on other hosts can be tested.
+type redirectTransport struct {
+ server *httptest.Server
+}
+
+func (t redirectTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ req = req.Clone(req.Context())
+ req.URL.Scheme = "http"
+ req.URL.Host = t.server.Listener.Addr().String()
+ return t.server.Client().Transport.RoundTrip(req)
+}
+
+// newDownloadTest returns an Object which downloads content from a
+// server which handles the /content call with contentHandler and
+// serves download.aspx with Range support, recording the
+// Authorization header of each download.
+func newDownloadTest(t *testing.T, content []byte, contentHandler http.HandlerFunc) (o *Object, auths *[]string) {
+ auths = new([]string)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/items/fake/content":
+ contentHandler(w, r)
+ case "/download.aspx":
+ *auths = append(*auths, r.Header.Get("Authorization"))
+ http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(content))
+ default:
+ t.Errorf("unexpected request %s %s", r.Method, r.URL)
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ t.Cleanup(server.Close)
+ transport := redirectTransport{server: server}
+ sharePointToken := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "sharepoint_token"})
+ sharePointHTTP := &http.Client{Transport: &oauth2.Transport{Source: sharePointToken, Base: transport}}
+ f := &Fs{
+ ci: fs.GetConfig(context.Background()),
+ srv: rest.NewClient(server.Client()).SetRoot(server.URL),
+ unAuth: rest.NewClient(&http.Client{Transport: transport}),
+ driveType: driveTypeBusiness,
+ tokenSource: &oauthutil.TokenSource{},
+ sharePoint: &sharePointClients{hosts: map[string]sharePointAuth{
+ "contoso-my.sharepoint.com": {client: rest.NewClient(sharePointHTTP), tokenSource: sharePointToken},
+ }},
+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+ }
+ f.opt.Region = "global"
+ return &Object{fs: f, id: "fake", size: int64(len(content))}, auths
+}
+
+// TestOpenRedirectAuth checks that downloads are redirected to a
+// pre-authenticated URL without authentication and to a SharePoint
+// URL with a SharePoint token.
+func TestOpenRedirectAuth(t *testing.T) {
+ content := []byte("0123456789abcdefghij")
+ for _, test := range []struct {
+ name string
+ location string
+ wantAuth string
+ }{
+ {"tempauth", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x&tempauth=v1.secret", ""},
+ {"sharepoint", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", "Bearer sharepoint_token"},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, test.location, http.StatusFound)
+ })
+ in, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})
+ require.NoError(t, err)
+ got, err := io.ReadAll(in)
+ require.NoError(t, err)
+ require.NoError(t, in.Close())
+ assert.Equal(t, "56789", string(got))
+ assert.Equal(t, []string{test.wantAuth}, *auths)
+ })
+ }
+
+ // Without credentials the error explains why
+ o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", http.StatusFound)
+ })
+ o.fs.tokenSource = nil
+ _, err := o.Open(context.Background())
+ assert.ErrorIs(t, err, errPreauthRetired)
+ assert.Empty(t, *auths)
+}
diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md
index d0f5f2d0f..12e6cacfd 100644
--- a/docs/content/onedrive.md
+++ b/docs/content/onedrive.md
@@ -1226,6 +1226,28 @@ rclone cleanup remote:path/subdir # unconditionally remove all old
## Troubleshooting
+### Pre-authenticated URLs
+
+Microsoft is [retiring pre-authenticated (tempauth) URLs](https://devblogs.microsoft.com/microsoft365dev/pre-authenticated-tempauth-urls-are-retiring-move-to-microsoft-entra-id-tokens-before-april-1-2027/)
+in OneDrive for Business and SharePoint from April 1, 2027.
+
+Rclone uses these URLs when downloading files, when uploading files
+bigger than `--onedrive-upload-cutoff` in chunks and when doing
+server-side copies. When SharePoint returns them without
+pre-authentication rclone authenticates them with a SharePoint token
+which it gets using the existing login, so no reconfiguration is
+needed. OneDrive Personal is not affected.
+
+If rclone can't get a SharePoint token, or SharePoint refuses it, you
+will see the error `SharePoint refused to authenticate a request`. In
+this case rclone uploads files smaller than 250 MiB in a single
+request instead and copies files by downloading and uploading them,
+but bigger files can't be uploaded and downloads may fail.
+
+SharePoint does not accept tokens from the client credentials flow
+which were got with a client secret, so this affects
+`client_credentials` users.
+
### Excessive throttling or blocked on SharePoint
If you experience excessive throttling or is being blocked on SharePoint then