"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
backend: pcloud: Implement OpenWriterAt feature
Georg Welzel4c1cb0622e39
diff --git a/backend/pcloud/api/types.go b/backend/pcloud/api/types.goindex c1b5dc217..6f2d6361b 100644--- a/backend/pcloud/api/types.go+++ b/backend/pcloud/api/types.go@@ -109,6 +109,37 @@ type Hashes struct {SHA256 string `json:"sha256"`}+// FileTruncateResponse is the response from /file_truncate+type FileTruncateResponse struct {+ Error+}++// FileCloseResponse is the response from /file_close+type FileCloseResponse struct {+ Error+}++// FileOpenResponse is the response from /file_open+type FileOpenResponse struct {+ Error+ Fileid int64 `json:"fileid"`+ FileDescriptor int64 `json:"fd"`+}++// FileChecksumResponse is the response from /file_checksum+type FileChecksumResponse struct {+ Error+ MD5 string `json:"md5"`+ SHA1 string `json:"sha1"`+ SHA256 string `json:"sha256"`+}++// FilePWriteResponse is the response from /file_pwrite+type FilePWriteResponse struct {+ Error+ Bytes int64 `json:"bytes"`+}+// UploadFileResponse is the response from /uploadfiletype UploadFileResponse struct {Errordiff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.goindex f17017cc1..dc475cb9d 100644--- a/backend/pcloud/pcloud.go+++ b/backend/pcloud/pcloud.go@@ -14,6 +14,7 @@ import ("net/http""net/url""path"+ "strconv""strings""time"@@ -146,7 +147,8 @@ we have to rely on user password authentication for it.`,Help: "Your pcloud password.",IsPassword: true,Advanced: true,- }}...),+ },+ }...),})}@@ -161,15 +163,16 @@ type Options struct {// Fs represents a remote pcloudtype Fs struct {- name string // name of this remote- root string // the path we are working on- opt Options // parsed options- features *fs.Features // optional features- srv *rest.Client // the connection to the server- cleanupSrv *rest.Client // the connection used for the cleanup method- dirCache *dircache.DirCache // Map of directory path to directory id- pacer *fs.Pacer // pacer for API calls- tokenRenewer *oauthutil.Renew // renew the token on expiry+ name string // name of this remote+ root string // the path we are working on+ opt Options // parsed options+ features *fs.Features // optional features+ ts *oauthutil.TokenSource // the token source, used to create new clients+ srv *rest.Client // the connection to the server+ cleanupSrv *rest.Client // the connection used for the cleanup method+ dirCache *dircache.DirCache // Map of directory path to directory id+ pacer *fs.Pacer // pacer for API calls+ tokenRenewer *oauthutil.Renew // renew the token on expiry}// Object describes a pcloud object@@ -317,6 +320,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ename: name,root: root,opt: *opt,+ ts: ts,srv: rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname),pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),}@@ -326,6 +330,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ef.features = (&fs.Features{CaseInsensitive: false,CanHaveEmptyDirectories: true,+ PartialUploads: true,}).Fill(ctx, f)if !canCleanup {f.features.CleanUp = nil@@ -333,7 +338,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ef.srv.SetErrorHandler(errorHandler)// Renew the token in the background- f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {+ f.tokenRenewer = oauthutil.NewRenew(f.String(), f.ts, func() error {_, err := f.readMetaDataForPath(ctx, "")return err})@@ -375,6 +380,56 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ereturn f, nil}+// OpenWriterAt opens with a handle for random access writes+//+// Pass in the remote desired and the size if known.+//+// It truncates any existing object+func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {+ client, err := f.newSingleConnClient(ctx)+ if err != nil {+ return nil, fmt.Errorf("create client: %w", err)+ }+ // init an empty file+ leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)+ if err != nil {+ return nil, fmt.Errorf("resolve src: %w", err)+ }+ openResult, err := fileOpenNew(ctx, client, f, directoryID, leaf)+ if err != nil {+ return nil, fmt.Errorf("open file: %w", err)+ }++ writer := &writerAt{+ ctx: ctx,+ client: client,+ fs: f,+ size: size,+ remote: remote,+ fd: openResult.FileDescriptor,+ fileID: openResult.Fileid,+ }++ return writer, nil+}++// Create a new http client, accepting keep-alive headers, limited to single connection.+// Necessary for pcloud fileops API, as it binds the session to the underlying TCP connection.+// File descriptors are only valid within the same connection and auto-closed when the connection is closed,+// hence we need a separate client (with single connection) for each fd to avoid all sorts of errors and race conditions.+func (f *Fs) newSingleConnClient(ctx context.Context) (*rest.Client, error) {+ baseClient := fshttp.NewClient(ctx)+ baseClient.Transport = fshttp.NewTransportCustom(ctx, func(t *http.Transport) {+ t.MaxConnsPerHost = 1+ t.DisableKeepAlives = false+ })+ // Set our own http client in the context+ ctx = oauthutil.Context(ctx, baseClient)+ // create a new oauth client, re-use the token source+ oAuthClient := oauth2.NewClient(ctx, f.ts)+ return rest.NewClient(oAuthClient).SetRoot("https://" + f.opt.Hostname), nil+}+// Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.@@ -1098,14 +1153,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {if err != nil {return err}- return o.setModTime(ctx, fileIDtoNumber(o.id), filename, directoryID, modTime)-}--func (o *Object) setModTime(- ctx context.Context,- fileID, filename, directoryID string,- modTime time.Time,-) error {+ fileID := fileIDtoNumber(o.id)filename = o.fs.opt.Enc.FromStandardName(filename)opts := rest.Opts{Method: "PUT",@@ -1124,7 +1172,7 @@ func (o *Object) setModTime(opts.Parameters.Set("mtime", strconv.FormatInt(modTime.Unix(), 10))result := &api.ItemResult{}- err := o.fs.pacer.CallNoRetry(func() (bool, error) {+ err = o.fs.pacer.CallNoRetry(func() (bool, error) {resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, result)err = result.Error.Update(err)return shouldRetry(ctx, resp, err)diff --git a/backend/pcloud/writer_at.go b/backend/pcloud/writer_at.gonew file mode 100644index 000000000..ee3380a71--- /dev/null+++ b/backend/pcloud/writer_at.go@@ -0,0 +1,216 @@+package pcloud++import (+ "bytes"+ "context"+ "crypto/sha1"+ "encoding/hex"+ "fmt"+ "net/url"+ "strconv"+ "time"++ "github.com/rclone/rclone/backend/pcloud/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/rest"+)++// writerAt implements fs.WriterAtCloser, adding the OpenWrtierAt feature to pcloud.+type writerAt struct {+ ctx context.Context+ client *rest.Client+ fs *Fs+ size int64+ remote string+ fd int64+ fileID int64+}++// Close implements WriterAt.Close.+func (c *writerAt) Close() error {+ // close fd+ if _, err := c.fileClose(c.ctx); err != nil {+ return fmt.Errorf("close fd: %w", err)+ }++ // Avoiding race conditions: Depending on the tcp connection, there might be+ // caching issues when checking the size immediately after write.+ // Hence we try avoiding them by checking the resulting size on a different connection.+ if c.size < 0 {+ // Without knowing the size, we cannot do size checks.+ // Falling back to a sleep of 1s for sake of hope.+ time.Sleep(1 * time.Second)+ return nil+ }+ sizeOk := false+ sizeLastSeen := int64(0)+ for retry := 0; retry < 5; retry++ {+ fs.Debugf(c.remote, "checking file size: try %d/5", retry)+ obj, err := c.fs.NewObject(c.ctx, c.remote)+ if err != nil {+ return fmt.Errorf("get uploaded obj: %w", err)+ }+ sizeLastSeen = obj.Size()+ if obj.Size() == c.size {+ sizeOk = true+ break+ }+ time.Sleep(1 * time.Second)+ }++ if !sizeOk {+ return fmt.Errorf("incorrect size after upload: got %d, want %d", sizeLastSeen, c.size)+ }++ return nil+}++// WriteAt implements fs.WriteAt.+func (c *writerAt) WriteAt(buffer []byte, offset int64) (n int, err error) {+ contentLength := len(buffer)++ inSHA1Bytes := sha1.Sum(buffer)+ inSHA1 := hex.EncodeToString(inSHA1Bytes[:])++ // get target hash+ outChecksum, err := c.fileChecksum(c.ctx, offset, int64(contentLength))+ if err != nil {+ return 0, err+ }+ outSHA1 := outChecksum.SHA1++ if outSHA1 == "" || inSHA1 == "" {+ return 0, fmt.Errorf("expect both hashes to be filled: src: %q, target: %q", inSHA1, outSHA1)+ }++ // check hash of buffer, skip if fits+ if inSHA1 == outSHA1 {+ return contentLength, nil+ }++ // upload buffer with offset if necessary+ if _, err := c.filePWrite(c.ctx, offset, buffer); err != nil {+ return 0, err+ }++ return contentLength, nil+}++// Call pcloud file_open using folderid and name with O_CREAT and O_WRITE flags, see [API Doc.]+// [API Doc]: https://docs.pcloud.com/methods/fileops/file_open.html+func fileOpenNew(ctx context.Context, c *rest.Client, srcFs *Fs, directoryID, filename string) (*api.FileOpenResponse, error) {+ opts := rest.Opts{+ Method: "PUT",+ Path: "/file_open",+ Parameters: url.Values{},+ TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding+ ExtraHeaders: map[string]string{+ "Connection": "keep-alive",+ },+ }+ filename = srcFs.opt.Enc.FromStandardName(filename)+ opts.Parameters.Set("name", filename)+ opts.Parameters.Set("folderid", dirIDtoNumber(directoryID))+ opts.Parameters.Set("flags", "0x0042") // O_CREAT, O_WRITE++ result := &api.FileOpenResponse{}+ err := srcFs.pacer.CallNoRetry(func() (bool, error) {+ resp, err := c.CallJSON(ctx, &opts, nil, result)+ err = result.Error.Update(err)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("open new file descriptor: %w", err)+ }+ return result, nil+}++// Call pcloud file_checksum, see [API Doc.]+// [API Doc]: https://docs.pcloud.com/methods/fileops/file_checksum.html+func (c *writerAt) fileChecksum(+ ctx context.Context,+ offset, count int64,+) (*api.FileChecksumResponse, error) {+ opts := rest.Opts{+ Method: "PUT",+ Path: "/file_checksum",+ Parameters: url.Values{},+ TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding+ ExtraHeaders: map[string]string{+ "Connection": "keep-alive",+ },+ }+ opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))+ opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))+ opts.Parameters.Set("count", strconv.FormatInt(count, 10))++ result := &api.FileChecksumResponse{}+ err := c.fs.pacer.CallNoRetry(func() (bool, error) {+ resp, err := c.client.CallJSON(ctx, &opts, nil, result)+ err = result.Error.Update(err)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("checksum of fd %d with offset %d and size %d: %w", c.fd, offset, count, err)+ }+ return result, nil+}++// Call pcloud file_pwrite, see [API Doc.]+// [API Doc]: https://docs.pcloud.com/methods/fileops/file_pwrite.html+func (c *writerAt) filePWrite(+ ctx context.Context,+ offset int64,+ buf []byte,+) (*api.FilePWriteResponse, error) {+ contentLength := int64(len(buf))+ opts := rest.Opts{+ Method: "PUT",+ Path: "/file_pwrite",+ Body: bytes.NewReader(buf),+ ContentLength: &contentLength,+ Parameters: url.Values{},+ TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding+ Close: false,+ ExtraHeaders: map[string]string{+ "Connection": "keep-alive",+ },+ }+ opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))+ opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))++ result := &api.FilePWriteResponse{}+ err := c.fs.pacer.CallNoRetry(func() (bool, error) {+ resp, err := c.client.CallJSON(ctx, &opts, nil, result)+ err = result.Error.Update(err)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("write %d bytes to fd %d with offset %d: %w", contentLength, c.fd, offset, err)+ }+ return result, nil+}++// Call pcloud file_close, see [API Doc.]+// [API Doc]: https://docs.pcloud.com/methods/fileops/file_close.html+func (c *writerAt) fileClose(ctx context.Context) (*api.FileCloseResponse, error) {+ opts := rest.Opts{+ Method: "PUT",+ Path: "/file_close",+ Parameters: url.Values{},+ TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding+ Close: true,+ }+ opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))++ result := &api.FileCloseResponse{}+ err := c.fs.pacer.CallNoRetry(func() (bool, error) {+ resp, err := c.client.CallJSON(ctx, &opts, nil, result)+ err = result.Error.Update(err)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("close file descriptor: %w", err)+ }+ return result, nil+}