Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

drime: fix large file uploads landing in drive root instead of configured folder

Nick Craig-Wood667903dca040
diff --git a/backend/drime/api/types.go b/backend/drime/api/types.go
index 9396034fa..9f70bf696 100644
--- a/backend/drime/api/types.go
+++ b/backend/drime/api/types.go
@@ -238,6 +238,14 @@ type MultiPartAbort struct {
Key string `json:"key"`
}
+// FolderPathResponse is returned by GET /folders/{hash}/path
+//
+// Path is the breadcrumb from the drive root down to the requested folder.
+type FolderPathResponse struct {
+ Status string `json:"status"`
+ Path []Item `json:"path"`
+}
+
// SpaceUsageResponse is returned by GET /user/space-usage
type SpaceUsageResponse struct {
Used int64 `json:"used"`
diff --git a/backend/drime/drime.go b/backend/drime/drime.go
index 3960c0d5f..2d6ae425f 100644
--- a/backend/drime/drime.go
+++ b/backend/drime/drime.go
@@ -15,6 +15,7 @@ should stay under that.
import (
"context"
+ "encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -199,13 +200,16 @@ type Options struct {
// Fs represents a remote drime
type Fs struct {
- name string // name of this remote
- root string // the path we are working on
- opt Options // parsed options
- features *fs.Features // optional features
- srv *rest.Client // the connection to the server
- dirCache *dircache.DirCache // Map of directory path to directory id
- pacer *fs.Pacer // pacer for API calls
+ name string // name of this remote
+ root string // the path we are working on
+ opt Options // parsed options
+ features *fs.Features // optional features
+ srv *rest.Client // the connection to the server
+ dirCache *dircache.DirCache // Map of directory path to directory id
+ pacer *fs.Pacer // pacer for API calls
+ absRootOnce *sync.Once // protects absRoot computation
+ absRoot string // absolute path of f.root from drive root
+ absRootErr error // error from computing absRoot, if any
}
// Object describes a drime object
@@ -330,6 +334,62 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
return info, err
}
+// idToAbsolutePath returns the absolute path (from the user's drive root) of
+// the folder with the given ID.
+//
+// Drime exposes GET /folders/{hash}/path which returns the breadcrumb from
+// drive root down to the folder. The hash format is undocumented but
+// observed to be base64("<id>|") - every item drime returns has a `hash`
+// field of that shape (e.g. id 704791396 => "NzA0NzkxMzk2fA"). The docs
+// example "MTExMzQ0fHBhZA" decodes to "111344|pad", so a non-empty suffix
+// after the pipe is also accepted; if drime ever starts requiring a
+// specific suffix this will break.
+//
+// Returns "" for an empty/zero ID (drive root).
+func (f *Fs) idToAbsolutePath(ctx context.Context, id string) (string, error) {
+ if id == "" || id == "0" {
+ return "", nil
+ }
+ hash := base64.StdEncoding.EncodeToString([]byte(id + "|"))
+ opts := rest.Opts{
+ Method: "GET",
+ Path: "/folders/" + hash + "/path",
+ Parameters: url.Values{},
+ }
+ if f.opt.WorkspaceID != "" {
+ opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
+ }
+ var result api.FolderPathResponse
+ var resp *http.Response
+ err := f.pacer.Call(func() (bool, error) {
+ var err error
+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return "", fmt.Errorf("failed to get folder path for %q: %w", id, err)
+ }
+ parts := make([]string, 0, len(result.Path))
+ for _, item := range result.Path {
+ parts = append(parts, f.opt.Enc.ToStandardName(item.Name))
+ }
+ return path.Join(parts...), nil
+}
+
+// absoluteRoot returns the absolute path from the drive root to the Fs root.
+// Computed once and cached.
+func (f *Fs) absoluteRoot(ctx context.Context) (string, error) {
+ f.absRootOnce.Do(func() {
+ rootID, err := f.dirCache.RootID(ctx, false)
+ if err != nil {
+ f.absRootErr = err
+ return
+ }
+ f.absRoot, f.absRootErr = f.idToAbsolutePath(ctx, rootID)
+ })
+ return f.absRoot, f.absRootErr
+}
+
// errorHandler parses a non 2xx error response into an error
func errorHandler(resp *http.Response) error {
body, err := rest.ReadBody(resp)
@@ -370,11 +430,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
client := fshttp.NewClient(ctx)
f := &Fs{
- name: name,
- root: root,
- opt: *opt,
- srv: rest.NewClient(client).SetRoot(rootURL),
- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ name: name,
+ root: root,
+ opt: *opt,
+ srv: rest.NewClient(client).SetRoot(rootURL),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ absRootOnce: new(sync.Once),
}
f.features = (&fs.Features{
CanHaveEmptyDirectories: true,
@@ -1096,6 +1157,15 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
return info, nil, err
}
+ // The /s3/multipart/create and /s3/entries endpoints interpret
+ // relativePath as an absolute path from the drive root, not relative to
+ // parent_id. Resolve our root's absolute path so we can build it.
+ absRoot, err := f.absoluteRoot(ctx)
+ if err != nil {
+ return info, nil, fmt.Errorf("failed to resolve absolute path of root: %w", err)
+ }
+ relPath := f.opt.Enc.FromStandardPath(path.Join(absRoot, remote))
+
// Temporary Object under construction
o := &Object{
fs: f,
@@ -1129,7 +1199,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
Size: createSize,
Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
ParentID: json.Number(directoryID),
- RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
+ RelativePath: relPath,
WorkspaceID: f.opt.WorkspaceID,
}
@@ -1156,8 +1226,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
if ext == "" {
ext = "bin"
}
- rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
-
chunkWriter := &drimeChunkWriter{
uploadID: resp.UploadID,
key: resp.Key,
@@ -1170,7 +1238,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
mime: mime,
extension: ext,
parentID: json.Number(directoryID),
- relativePath: rel,
+ relativePath: relPath,
}
info = fs.ChunkWriterInfo{
ChunkSize: int64(chunkSize),