Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

zoho: fix inefficiencies uploading with new API to avoid throttling

Nick Craig-Wood802a938bd1c5
diff --git a/backend/zoho/api/types.go b/backend/zoho/api/types.go
index 5a516b6fb..ed6751a44 100644
--- a/backend/zoho/api/types.go
+++ b/backend/zoho/api/types.go
@@ -2,6 +2,8 @@
package api
import (
+ "encoding/json"
+ "fmt"
"strconv"
"time"
)
@@ -12,7 +14,12 @@ type Time time.Time
// UnmarshalJSON turns JSON into a Time
func (t *Time) UnmarshalJSON(data []byte) error {
- millis, err := strconv.ParseInt(string(data), 10, 64)
+ s := string(data)
+ // If the time is a quoted string, strip quotes
+ if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
+ s = s[1 : len(s)-1]
+ }
+ millis, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return err
}
@@ -84,6 +91,73 @@ type ItemList struct {
Items []Item `json:"data"`
}
+// UploadFileInfo is what the FileInfo field in the UnloadInfo struct decodes to
+type UploadFileInfo struct {
+ OrgID string `json:"ORG_ID"`
+ ResourceID string `json:"RESOURCE_ID"`
+ LibraryID string `json:"LIBRARY_ID"`
+ Md5Checksum string `json:"MD5_CHECKSUM"`
+ ParentModelID string `json:"PARENT_MODEL_ID"`
+ ParentID string `json:"PARENT_ID"`
+ ResourceType int `json:"RESOURCE_TYPE"`
+ WmsSentTime string `json:"WMS_SENT_TIME"`
+ TabID string `json:"TAB_ID"`
+ Owner string `json:"OWNER"`
+ ResourceGroup string `json:"RESOURCE_GROUP"`
+ ParentModelName string `json:"PARENT_MODEL_NAME"`
+ Size int64 `json:"size"`
+ Operation string `json:"OPERATION"`
+ EventID string `json:"EVENT_ID"`
+ AuditInfo struct {
+ VersionInfo struct {
+ VersionAuthors []string `json:"versionAuthors"`
+ VersionID string `json:"versionId"`
+ IsMinorVersion bool `json:"isMinorVersion"`
+ VersionTime Time `json:"versionTime"`
+ VersionAuthorZuid []string `json:"versionAuthorZuid"`
+ VersionNotes string `json:"versionNotes"`
+ VersionNumber string `json:"versionNumber"`
+ } `json:"versionInfo"`
+ Resource struct {
+ Owner string `json:"owner"`
+ CreatedTime Time `json:"created_time"`
+ Creator string `json:"creator"`
+ ServiceType int `json:"service_type"`
+ Extension string `json:"extension"`
+ StatusChangeTime Time `json:"status_change_time"`
+ ResourceType int `json:"resource_type"`
+ Name string `json:"name"`
+ } `json:"resource"`
+ ParentInfo struct {
+ ParentName string `json:"parentName"`
+ ParentID string `json:"parentId"`
+ ParentType int `json:"parentType"`
+ } `json:"parentInfo"`
+ LibraryInfo struct {
+ LibraryName string `json:"libraryName"`
+ LibraryID string `json:"libraryId"`
+ LibraryType int `json:"libraryType"`
+ } `json:"libraryInfo"`
+ UpdateType string `json:"updateType"`
+ StatusCode string `json:"statusCode"`
+ } `json:"AUDIT_INFO"`
+ ZUID int64 `json:"ZUID"`
+ TeamID string `json:"TEAM_ID"`
+}
+
+// GetModTime fetches the modification time of the upload
+//
+// This tries a few places and if all fails returns the current time
+func (ufi *UploadFileInfo) GetModTime() Time {
+ if t := ufi.AuditInfo.Resource.CreatedTime; !time.Time(t).IsZero() {
+ return t
+ }
+ if t := ufi.AuditInfo.Resource.StatusChangeTime; !time.Time(t).IsZero() {
+ return t
+ }
+ return Time(time.Now())
+}
+
// UploadInfo is a simplified and slightly different version of
// the Item struct only used in the response to uploads
type UploadInfo struct {
@@ -91,9 +165,21 @@ type UploadInfo struct {
ParentID string `json:"parent_id"`
FileName string `json:"notes.txt"`
RessourceID string `json:"resource_id"`
+ Permalink string `json:"Permalink"`
+ FileInfo string `json:"File INFO"` // JSON encoded UploadFileInfo
} `json:"attributes"`
}
+// GetUploadFileInfo decodes the embedded FileInfo
+func (ui *UploadInfo) GetUploadFileInfo() (*UploadFileInfo, error) {
+ var ufi UploadFileInfo
+ err := json.Unmarshal([]byte(ui.Attributes.FileInfo), &ufi)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode FileInfo: %w", err)
+ }
+ return &ufi, nil
+}
+
// UploadResponse is the response to a file Upload
type UploadResponse struct {
Uploads []UploadInfo `json:"data"`
diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go
index e4e69353f..207289c71 100644
--- a/backend/zoho/zoho.go
+++ b/backend/zoho/zoho.go
@@ -677,25 +677,26 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
if len(uploadResponse.Uploads) != 1 {
return nil, errors.New("upload: invalid response")
}
- // Received meta data is missing size so we have to read it again.
- // It doesn't always appear on first read so try again if necessary
- var info *api.Item
- const maxTries = 10
- sleepTime := 100 * time.Millisecond
- for i := 0; i < maxTries; i++ {
- info, err = f.readMetaDataForID(ctx, uploadResponse.Uploads[0].Attributes.RessourceID)
- if err != nil {
- return nil, err
- }
- if info.Attributes.StorageInfo.Size != 0 || size == 0 {
- break
- }
- fs.Debugf(f, "Size not available yet for %q - try again in %v (try %d/%d)", name, sleepTime, i+1, maxTries)
- time.Sleep(sleepTime)
- sleepTime *= 2
+ upload := uploadResponse.Uploads[0]
+ uploadInfo, err := upload.GetUploadFileInfo()
+ if err != nil {
+ return nil, fmt.Errorf("upload error: %w", err)
}
- return info, nil
+ // Fill in the api.Item from the api.UploadFileInfo
+ var info api.Item
+ info.ID = upload.Attributes.RessourceID
+ info.Attributes.Name = upload.Attributes.FileName
+ // info.Attributes.Type = not used
+ info.Attributes.IsFolder = false
+ // info.Attributes.CreatedTime = not used
+ info.Attributes.ModifiedTime = uploadInfo.GetModTime()
+ // info.Attributes.UploadedTime = 0 not used
+ info.Attributes.StorageInfo.Size = uploadInfo.Size
+ info.Attributes.StorageInfo.FileCount = 0
+ info.Attributes.StorageInfo.FolderCount = 0
+
+ return &info, nil
}
// Put the object into the container