Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

Add new backend dosya for dosya.dev cloud storage

Kaya408cc55171b6
diff --git a/README.md b/README.md
index e4456493f..f1eab0da4 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,7 @@ directories to and from different cloud storage providers.
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
+- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
diff --git a/backend/all/all.go b/backend/all/all.go
index 37c8f56e9..0d8bed8c8 100644
--- a/backend/all/all.go
+++ b/backend/all/all.go
@@ -16,6 +16,7 @@ import (
_ "github.com/rclone/rclone/backend/compress"
_ "github.com/rclone/rclone/backend/crypt"
_ "github.com/rclone/rclone/backend/doi"
+ _ "github.com/rclone/rclone/backend/dosya"
_ "github.com/rclone/rclone/backend/drime"
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/dropbox"
diff --git a/backend/dosya/api/types.go b/backend/dosya/api/types.go
new file mode 100644
index 000000000..5d0cb89a7
--- /dev/null
+++ b/backend/dosya/api/types.go
@@ -0,0 +1,259 @@
+// Package api has type definitions for dosya.dev
+package api
+
+import "fmt"
+
+// Response is the base response wrapper
+type Response struct {
+ OK bool `json:"ok"`
+ Error string `json:"error,omitempty"`
+}
+
+// Error is an error response from the API.
+//
+// StatusCode and Status are filled in from the HTTP response. The API
+// sets ErrorCode on the errors a client is expected to act on rather
+// than just report, e.g. "concurrent_upload_limit".
+type Error struct {
+ StatusCode int `json:"-"`
+ Status string `json:"-"`
+ OK bool `json:"ok"`
+ Message string `json:"error"`
+ ErrorCode string `json:"error_code,omitempty"`
+ MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
+}
+
+// Error satisfies the error interface
+func (e *Error) Error() string {
+ out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
+ if e.Message != "" {
+ out += ": " + e.Message
+ }
+ if e.ErrorCode != "" {
+ out += " (" + e.ErrorCode + ")"
+ }
+ return out
+}
+
+// FileItem represents a file from the dosya API
+type FileItem struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ SizeBytes int64 `json:"size_bytes"`
+ MimeType string `json:"mime_type"`
+ Extension string `json:"extension"`
+ Region string `json:"region"`
+ CreatedAt float64 `json:"created_at"`
+ UpdatedAt float64 `json:"updated_at"`
+ FolderID *string `json:"folder_id"`
+ IsSynced int `json:"is_synced"`
+ // ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
+ // the server has none (e.g. a file uploaded as an R2 multipart, whose parts
+ // are never re-read to hash the whole object - the same limitation S3 has).
+ ContentHash string `json:"content_hash"`
+}
+
+// FolderItem represents a folder from the dosya API
+type FolderItem struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ ParentID *string `json:"parent_id"`
+ CreatedAt float64 `json:"created_at"`
+ FileCount int `json:"file_count"`
+ IsSynced int `json:"is_synced"`
+}
+
+// ListResponse is the response from GET /api/files
+type ListResponse struct {
+ Response
+ Files []FileItem `json:"files"`
+ Folders []FolderItem `json:"folders"`
+ Pagination Pagination `json:"pagination"`
+}
+
+// Pagination describes which page of files a ListResponse holds
+type Pagination struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+ TotalFiles int `json:"total_files"`
+ TotalPages int `json:"total_pages"`
+}
+
+// CreateFolderRequest is the request body for POST /api/folders
+type CreateFolderRequest struct {
+ WorkspaceID string `json:"workspace_id"`
+ ParentID *string `json:"parent_id"`
+ Name string `json:"name"`
+}
+
+// CreateFolderResponse is the response from POST /api/folders
+type CreateFolderResponse struct {
+ Response
+ Folder struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ ParentID *string `json:"parent_id"`
+ Workspace string `json:"workspace_id"`
+ } `json:"folder"`
+}
+
+// DeleteFolderResponse is the response from DELETE /api/folders/:id
+type DeleteFolderResponse struct {
+ Response
+ Permanent bool `json:"permanent"`
+ Complete bool `json:"complete"`
+ Remaining int `json:"remaining"`
+ FilesAffected int `json:"files_affected"`
+ FoldersRemoved int `json:"folders_removed"`
+}
+
+// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
+type RenameFolderRequest struct {
+ Name string `json:"name"`
+}
+
+// MoveFolderRequest is the request body for PUT /api/folders/:id/move
+type MoveFolderRequest struct {
+ ParentID *string `json:"parent_id"`
+ Name string `json:"name,omitempty"`
+}
+
+// MoveFolderResponse is the response from PUT /api/folders/:id/move
+type MoveFolderResponse struct {
+ Response
+ Name string `json:"name"`
+}
+
+// UploadInitRequest is the request body for POST /api/upload/init
+type UploadInitRequest struct {
+ WorkspaceID string `json:"workspace_id"`
+ FileName string `json:"file_name"`
+ FileSize int64 `json:"file_size"`
+ MimeType string `json:"mime_type"`
+ Region string `json:"region,omitempty"`
+ FolderID *string `json:"folder_id"`
+ FileID *string `json:"file_id,omitempty"`
+}
+
+// UploadInitResponse is the response from POST /api/upload/init
+type UploadInitResponse struct {
+ Response
+ SessionID string `json:"session_id"`
+ UploadURL string `json:"upload_url"`
+ WorkspaceID string `json:"workspace_id"`
+ FileName string `json:"file_name"`
+ FileSize int64 `json:"file_size"`
+ MimeType string `json:"mime_type"`
+ Extension string `json:"extension"`
+ Region string `json:"region"`
+ Resumable *struct {
+ PartSize int64 `json:"part_size"`
+ TotalParts int `json:"total_parts"`
+ PartUploadURL string `json:"part_upload_url"`
+ CompleteURL string `json:"complete_url"`
+ StatusURL string `json:"status_url"`
+ } `json:"resumable"`
+}
+
+// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
+type UploadCompleteResponse struct {
+ Response
+ File struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ SizeBytes int64 `json:"size_bytes"`
+ MimeType string `json:"mime_type"`
+ Extension string `json:"extension"`
+ Region string `json:"region"`
+ Version int `json:"version"`
+ CreatedAt float64 `json:"created_at"`
+ // SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
+ // "" for a multipart upload, which the server does not hash.
+ ContentHash string `json:"content_hash"`
+ } `json:"file"`
+}
+
+// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
+type UploadPartResponse struct {
+ Response
+ PartNumber int `json:"part_number"`
+ ETag string `json:"etag"`
+ BytesUploaded int64 `json:"bytes_uploaded"`
+ PartsCompleted int `json:"parts_completed"`
+ TotalParts int `json:"total_parts"`
+}
+
+// DeleteFileResponse is the response from DELETE /api/files/:id
+type DeleteFileResponse struct {
+ Response
+ Permanent bool `json:"permanent"`
+}
+
+// RenameFileRequest is the request body for PUT /api/files/:id/rename
+type RenameFileRequest struct {
+ Name string `json:"name"`
+}
+
+// RenameFileResponse is the response from PUT /api/files/:id/rename
+type RenameFileResponse struct {
+ Response
+ Name string `json:"name"`
+}
+
+// MoveFileRequest is the request body for PUT /api/files/:id/move
+type MoveFileRequest struct {
+ FolderID *string `json:"folder_id"`
+ Name *string `json:"name,omitempty"`
+}
+
+// CopyFileRequest is the request body for POST /api/files/:id/copy
+type CopyFileRequest struct {
+ FolderID *string `json:"folder_id"`
+ Name string `json:"name,omitempty"`
+}
+
+// CopyFileResponse is the response from POST /api/files/:id/copy
+type CopyFileResponse struct {
+ Response
+ FileID string `json:"file_id"`
+ Name string `json:"name"`
+}
+
+// ShareLinkRequest is the request body for POST /api/files/:id/share
+type ShareLinkRequest struct {
+ ExpiresInDays *int `json:"expires_in_days,omitempty"`
+}
+
+// ShareLinkResponse is the response from POST /api/files/:id/share
+type ShareLinkResponse struct {
+ Response
+ Link struct {
+ ID string `json:"id"`
+ Token string `json:"token"`
+ URL string `json:"url"`
+ ExpiresAt *int64 `json:"expires_at"`
+ CreatedAt float64 `json:"created_at"`
+ } `json:"link"`
+}
+
+// Workspace is one entry of the workspace list
+type Workspace struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+}
+
+// WorkspaceListResponse is the response from GET /api/workspaces
+type WorkspaceListResponse struct {
+ Response
+ Workspaces []Workspace `json:"workspaces"`
+}
+
+// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
+type WorkspaceInfoResponse struct {
+ Response
+ Storage struct {
+ Used int64 `json:"used"`
+ Total int64 `json:"total"`
+ Free int64 `json:"free"`
+ } `json:"storage"`
+}
diff --git a/backend/dosya/dosya.go b/backend/dosya/dosya.go
new file mode 100644
index 000000000..4256d4674
--- /dev/null
+++ b/backend/dosya/dosya.go
@@ -0,0 +1,772 @@
+// Package dosya provides an interface to the dosya.dev cloud storage system.
+package dosya
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+ "time"
+
+ "github.com/rclone/rclone/backend/dosya/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/config/configstruct"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/lib/dircache"
+ "github.com/rclone/rclone/lib/encoder"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/rclone/rclone/lib/rest"
+)
+
+const (
+ rootID = "root"
+ defaultBaseURL = "https://api.dosya.dev"
+ minSleep = 100 * time.Millisecond
+ maxSleep = 2 * time.Second
+ decayConstant = 2
+ attackConstant = 0
+)
+
+func init() {
+ fs.Register(&fs.RegInfo{
+ Name: "dosya",
+ Description: "dosya.dev",
+ NewFs: NewFs,
+ Options: []fs.Option{{
+ Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
+ Name: "api_key",
+ Sensitive: true,
+ Required: true,
+ }, {
+ Name: "workspace_id",
+ Help: `Your workspace ID.
+
+Leave this blank to use the one workspace the API key can reach, which
+is what a key restricted to a single workspace does. If the key can
+reach several, rclone will refuse and list their IDs for you to choose
+from.`,
+ }, {
+ Help: "API base URL.",
+ Name: "api_url",
+ Default: defaultBaseURL,
+ Advanced: true,
+ }, {
+ Name: "upload_concurrency",
+ Help: `Concurrency for multipart uploads.
+
+This is the number of parts of the same file that are uploaded
+concurrently. The first part is always uploaded on its own, because
+the server creates the upload when it receives it.
+
+Each part in flight is held in memory, so memory use grows with
+this value times the part size the server chooses.
+
+If you are uploading large files over high-speed links and these
+uploads do not fully utilize your bandwidth, then increasing this
+may help to speed up the transfers.`,
+ Default: 4,
+ Advanced: true,
+ }, {
+ Name: config.ConfigEncoding,
+ Help: config.ConfigEncodingHelp,
+ Advanced: true,
+ Default: (encoder.Display |
+ encoder.EncodeBackSlash |
+ encoder.EncodeSingleQuote |
+ encoder.EncodeDoubleQuote |
+ encoder.EncodeLtGt |
+ encoder.EncodeCtl |
+ encoder.EncodeLeftSpace |
+ encoder.EncodeRightSpace |
+ encoder.EncodeInvalidUtf8),
+ }},
+ })
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+ APIKey string `config:"api_key"`
+ WorkspaceID string `config:"workspace_id"`
+ APIURL string `config:"api_url"`
+ UploadConcurrency int `config:"upload_concurrency"`
+ Enc encoder.MultiEncoder `config:"encoding"`
+}
+
+// Fs represents a remote dosya.dev filesystem
+type Fs struct {
+ name string
+ root string
+ opt Options
+ features *fs.Features
+ rest *rest.Client
+ pacer *fs.Pacer
+ dirCache *dircache.DirCache
+}
+
+// Object represents a dosya.dev file
+type Object struct {
+ fs *Fs
+ remote string
+ file api.FileItem
+}
+
+// findWorkspace fills in the workspace ID when the config leaves it blank.
+//
+// The ID is not shown anywhere in the dosya.dev web interface, so requiring
+// it meant asking people to find it in a developer console. A key restricted
+// to one workspace can only mean that one, and an account with a single
+// workspace has only one possible answer; anything else genuinely has to be
+// chosen, so list the choices in the error.
+func (f *Fs) findWorkspace(ctx context.Context) error {
+ workspaces, err := f.listWorkspaces(ctx)
+ if err != nil {
+ return err
+ }
+ switch len(workspaces) {
+ case 0:
+ return errors.New("no workspaces found for this API key")
+ case 1:
+ f.opt.WorkspaceID = workspaces[0].ID
+ fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
+ return nil
+ }
+ var choices strings.Builder
+ for _, w := range workspaces {
+ _, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
+ }
+ return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
+ len(workspaces), choices.String())
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathID
+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
+ listing, err := f.listFilesAndFolders(ctx, pathID)
+ if err != nil {
+ return "", false, err
+ }
+ for _, folder := range listing.Folders {
+ if f.opt.Enc.ToStandardName(folder.Name) == leaf {
+ return folder.ID, true, nil
+ }
+ }
+ return "", false, nil
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+ name := f.opt.Enc.FromStandardName(leaf)
+ resp, err := f.createFolder(ctx, name, pathID)
+ if err != nil {
+ return "", err
+ }
+ return resp.Folder.ID, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+ return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+ return f.root
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+ return fmt.Sprintf("dosya.dev root '%s'", f.root)
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+ // Modification times are stored and returned to the second (the API
+ // keeps them as Unix seconds, sent on upload via the source-mtime header).
+ return time.Second
+}
+
+// Hashes returns the supported hash types of the filesystem
+func (f *Fs) Hashes() hash.Set {
+ // The server computes and stores the SHA-256 of a single-shot upload's
+ // bytes; multipart uploads have none (see Object.Hash).
+ return hash.NewHashSet(hash.SHA256)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+ return f.features
+}
+
+// NewFs makes a new Fs object from the path
+func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
+ opt := new(Options)
+ err := configstruct.Set(config, opt)
+ if err != nil {
+ return nil, err
+ }
+
+ root = strings.Trim(root, "/")
+
+ f := &Fs{
+ name: name,
+ root: root,
+ opt: *opt,
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
+ }
+
+ // No DuplicateFiles: an upload that names no existing file but whose
+ // (workspace, folder, name) is already taken becomes a new VERSION of
+ // that file rather than a second row, so a name is unique within a
+ // folder and the same upload twice returns the same file id.
+ f.features = (&fs.Features{
+ CaseInsensitive: false,
+ CanHaveEmptyDirectories: true,
+ ReadMimeType: true,
+ }).Fill(ctx, f)
+
+ client := fshttp.NewClient(ctx)
+ f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
+ f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
+
+ if f.opt.WorkspaceID == "" {
+ err = f.findWorkspace(ctx)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ f.dirCache = dircache.New(root, rootID, f)
+
+ // Find the current root
+ err = f.dirCache.FindRoot(ctx, false)
+ if err != nil {
+ // Assume it is a file
+ newRoot, remote := dircache.SplitPath(root)
+ tempF := *f
+ tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
+ tempF.root = newRoot
+ // Make new Fs which is the parent
+ err = tempF.dirCache.FindRoot(ctx, false)
+ if err != nil {
+ // No root so return old f
+ return f, nil
+ }
+ _, err := tempF.NewObject(ctx, remote)
+ if err != nil {
+ if err == fs.ErrorObjectNotFound {
+ // File doesn't exist so return old f
+ return f, nil
+ }
+ return nil, err
+ }
+ f.features.Fill(ctx, &tempF)
+ f.dirCache = tempF.dirCache
+ f.root = tempF.root
+ return f, fs.ErrorIsFile
+ }
+ return f, nil
+}
+
+// List the objects and directories in dir into entries
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+ if err != nil {
+ return nil, err
+ }
+
+ listing, err := f.listFilesAndFolders(ctx, directoryID)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, folder := range listing.Folders {
+ folderName := f.opt.Enc.ToStandardName(folder.Name)
+ fullPath := getRemote(dir, folderName)
+ modTime := time.Unix(int64(folder.CreatedAt), 0)
+ d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
+ entries = append(entries, d)
+ f.dirCache.Put(fullPath, folder.ID)
+ }
+
+ for _, file := range listing.Files {
+ fileName := f.opt.Enc.ToStandardName(file.Name)
+ remote := getRemote(dir, fileName)
+ o := &Object{
+ fs: f,
+ remote: remote,
+ file: file,
+ }
+ entries = append(entries, o)
+ }
+
+ return entries, nil
+}
+
+// NewObject finds the Object at remote
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+ leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
+ if err != nil {
+ if err == fs.ErrorDirNotFound {
+ return nil, fs.ErrorObjectNotFound
+ }
+ return nil, err
+ }
+
+ listing, err := f.listFilesAndFolders(ctx, directoryID)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, file := range listing.Files {
+ if f.opt.Enc.ToStandardName(file.Name) == leaf {
+ return &Object{
+ fs: f,
+ remote: remote,
+ file: file,
+ }, nil
+ }
+ }
+
+ return nil, fs.ErrorObjectNotFound
+}
+
+// Put uploads a new file
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ existingObj, err := f.NewObject(ctx, src.Remote())
+ switch err {
+ case nil:
+ return existingObj, existingObj.Update(ctx, in, src, options...)
+ case fs.ErrorObjectNotFound:
+ return f.put(ctx, in, src, options...)
+ default:
+ return nil, err
+ }
+}
+
+// put uploads src as a new file
+func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ remote := src.Remote()
+
+ _, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
+ if err != nil {
+ return nil, err
+ }
+
+ modTime := src.ModTime(ctx)
+ resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ return &Object{
+ fs: f,
+ remote: remote,
+ file: fileFromUpload(resp, modTime),
+ }, nil
+}
+
+// fileFromUpload builds the FileItem for the object an upload just created.
+// The single-shot upload response omits updated_at, so the modification time
+// is taken from what was sent (modTime) rather than defaulting to the create
+// time - it is the time the server stored via the source-mtime header. The
+// SHA-256 comes straight from the response (empty for a multipart upload).
+func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
+ updatedAt := resp.File.CreatedAt
+ if !modTime.IsZero() {
+ updatedAt = float64(modTime.Unix())
+ }
+ return api.FileItem{
+ ID: resp.File.ID,
+ Name: resp.File.Name,
+ SizeBytes: resp.File.SizeBytes,
+ MimeType: resp.File.MimeType,
+ Extension: resp.File.Extension,
+ Region: resp.File.Region,
+ CreatedAt: resp.File.CreatedAt,
+ UpdatedAt: updatedAt,
+ ContentHash: resp.File.ContentHash,
+ }
+}
+
+// Mkdir makes the directory (container, bucket)
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+ _, err := f.dirCache.FindDir(ctx, dir, true)
+ return err
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+ if err != nil {
+ return err
+ }
+
+ // The server checks emptiness itself (?if_empty=1) and answers
+ // fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
+ err = f.removeFolder(ctx, directoryID, true)
+ if err != nil {
+ return err
+ }
+
+ f.dirCache.FlushDir(dir)
+ return nil
+}
+
+// Move src to this remote using server-side move operations
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+ srcObj, ok := src.(*Object)
+ if !ok {
+ fs.Debugf(src, "Can't move - not same remote type")
+ return nil, fs.ErrorCantMove
+ }
+
+ srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
+ _, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
+ if err != nil {
+ return nil, err
+ }
+
+ // Find destination directory
+ dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
+ if err != nil {
+ return nil, err
+ }
+
+ needsMove := srcDirID != dstDirID
+ needsRename := srcLeaf != dstLeaf
+
+ // Moving and renaming in the same request (rather than a moveFile call
+ // followed by a separate renameFile call) matters when both are needed:
+ // a two-step move-then-rename can transiently place the file's CURRENT
+ // name into a destination folder that already holds an unrelated file
+ // with that name, even though the FINAL name is free - the server only
+ // allows one active name per folder, so that intermediate state 409s.
+ if needsMove || needsRename {
+ newName := ""
+ if needsRename {
+ newName = f.opt.Enc.FromStandardName(dstLeaf)
+ }
+ err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Return new object. file.Name must reflect the destination leaf, not
+ // the source's - a caller that reuses this object as the src of a
+ // SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
+ // recomputes srcLeaf from file.Name; a stale name there makes that
+ // second move see srcLeaf == dstLeaf and skip the rename entirely,
+ // silently leaving the object under the wrong name.
+ newFile := srcObj.file
+ newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
+ return &Object{
+ fs: f,
+ remote: remote,
+ file: newFile,
+ }, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+ srcFs, ok := src.(*Fs)
+ if !ok {
+ fs.Debugf(srcFs, "Can't move directory - not same remote type")
+ return fs.ErrorCantDirMove
+ }
+
+ srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
+ if err != nil {
+ return err
+ }
+
+ srcLeaf := srcRemote
+ if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
+ srcLeaf = srcRemote[idx+1:]
+ }
+ needsRename := srcLeaf != dstLeaf
+ newName := ""
+ if needsRename {
+ newName = f.opt.Enc.FromStandardName(dstLeaf)
+ }
+
+ // Move folder to new parent only if parent changed. The new name goes
+ // in the same request, as for files in Move: a move followed by a
+ // separate rename can collide with an unrelated folder sharing the
+ // CURRENT name in the destination even though the final name is free.
+ if srcDirectoryID != dstDirectoryID {
+ renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
+ if err != nil {
+ return fmt.Errorf("couldn't move directory: %w", err)
+ }
+ if renamed {
+ needsRename = false
+ }
+ }
+
+ if needsRename {
+ err = f.renameFolder(ctx, srcID, newName)
+ if err != nil {
+ return fmt.Errorf("couldn't rename directory: %w", err)
+ }
+ }
+
+ srcFs.dirCache.FlushDir(srcRemote)
+ return nil
+}
+
+// Copy src to this remote using server-side copy operations
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+ srcObj, ok := src.(*Object)
+ if !ok {
+ fs.Debugf(src, "Can't copy - not same remote type")
+ return nil, fs.ErrorCantCopy
+ }
+
+ dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
+ if err != nil {
+ return nil, err
+ }
+
+ // The copy API creates a new row named after the source file when
+ // copying across folders, which collides with one already at the
+ // destination. Copy under a temporary name, then delete the old
+ // destination, then move the copy onto the name - so a copy that fails
+ // leaves the destination as it was. Deleting it up front lost it
+ // (prior art: backend/box).
+ existingObj, err := f.NewObject(ctx, remote)
+ switch err {
+ case nil:
+ tempRemote := remote + "-rclone-copy-" + random.String(8)
+ fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
+ tempObj, err := f.Copy(ctx, src, tempRemote)
+ if err != nil {
+ return nil, err
+ }
+ fs.Debugf(tempRemote, "moving to real name %v", remote)
+ if err := existingObj.Remove(ctx); err != nil {
+ return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
+ }
+ return f.Move(ctx, tempObj, remote)
+ case fs.ErrorObjectNotFound:
+ // nothing at the destination, so copy straight onto the name
+ default:
+ return nil, err
+ }
+
+ dstName := f.opt.Enc.FromStandardName(dstLeaf)
+
+ resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't copy file: %w", err)
+ }
+
+ // Servers that predate the name field ignore it and name the copy after
+ // the source, or "Copy of <original>" in the same folder. Rename to the
+ // requested destination name if it differs.
+ if resp.Name != dstName {
+ err = f.renameFile(ctx, resp.FileID, dstName)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't rename copied file: %w", err)
+ }
+ resp.Name = dstName
+ }
+
+ // A server-side copy is byte-identical to the source and the API
+ // preserves the source's modification time and content hash onto the
+ // copy, so the returned object must report the SOURCE's modtime and hash,
+ // not the copy's wall-clock time. Reading "now" here made the object
+ // rclone holds in memory disagree with the one read back from the remote
+ // (its Fingerprint check compares the two exactly).
+ return &Object{
+ fs: f,
+ remote: remote,
+ file: api.FileItem{
+ ID: resp.FileID,
+ Name: resp.Name,
+ SizeBytes: srcObj.file.SizeBytes,
+ MimeType: srcObj.file.MimeType,
+ Extension: srcObj.file.Extension,
+ Region: srcObj.file.Region,
+ CreatedAt: srcObj.file.CreatedAt,
+ UpdatedAt: srcObj.file.UpdatedAt,
+ ContentHash: srcObj.file.ContentHash,
+ },
+ }, nil
+}
+
+// Purge deletes all the files and the container
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+ if err != nil {
+ return err
+ }
+
+ // DELETE /api/folders/:id?permanent=1 recursively deletes all contents
+ err = f.removeFolder(ctx, directoryID, false)
+ if err != nil {
+ return err
+ }
+
+ f.dirCache.FlushDir(dir)
+ return nil
+}
+
+// PublicLink generates a public link to the remote path
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+ o, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return "", err
+ }
+ obj := o.(*Object)
+
+ resp, err := f.createShareLink(ctx, obj.file.ID, expire)
+ if err != nil {
+ return "", err
+ }
+
+ return resp.Link.URL, nil
+}
+
+// DirCacheFlush resets the directory cache - used in testing as an
+// optional interface
+func (f *Fs) DirCacheFlush() {
+ f.dirCache.ResetRoot()
+}
+
+// About gets quota information from the Fs
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+ info, err := f.getWorkspaceInfo(ctx)
+ if err != nil {
+ return nil, err
+ }
+ usage := &fs.Usage{
+ Used: fs.NewUsageValue(info.Storage.Used),
+ Total: fs.NewUsageValue(info.Storage.Total),
+ Free: fs.NewUsageValue(info.Storage.Free),
+ }
+ return usage, nil
+}
+
+// ------------------------------------------------------------
+// Object methods
+// ------------------------------------------------------------
+
+// String returns a description of the Object
+func (o *Object) String() string {
+ return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+ return o.remote
+}
+
+// ModTime returns the modification date of the file
+func (o *Object) ModTime(ctx context.Context) time.Time {
+ if o.file.UpdatedAt > 0 {
+ return time.Unix(int64(o.file.UpdatedAt), 0)
+ }
+ return time.Unix(int64(o.file.CreatedAt), 0)
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+ return o.file.SizeBytes
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+ return o.fs
+}
+
+// Hash returns the selected checksum of the file
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+ if t != hash.SHA256 {
+ return "", hash.ErrUnsupported
+ }
+ // May be "" - a multipart-uploaded object carries no server-side hash,
+ // which rclone treats as "hash unavailable for this object" (as it does
+ // for S3 multipart objects).
+ return o.file.ContentHash, nil
+}
+
+// Storable says whether this object can be stored
+func (o *Object) Storable() bool {
+ return true
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+ return fs.ErrorCantSetModTime
+}
+
+// Open opens the file for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+ fs.FixRangeOption(options, o.file.SizeBytes)
+ return o.fs.downloadFile(ctx, o.file.ID, options)
+}
+
+// Update replaces the file content
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+ // Find the directory ID for this file
+ _, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
+ if err != nil {
+ return err
+ }
+
+ // Upload as new version of existing file
+ fileID := o.file.ID
+ modTime := src.ModTime(ctx)
+ resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
+ if err != nil {
+ return err
+ }
+
+ // Update object metadata
+ o.file = fileFromUpload(resp, modTime)
+
+ return nil
+}
+
+// Remove removes this object
+func (o *Object) Remove(ctx context.Context) error {
+ return o.fs.deleteFile(ctx, o.file.ID)
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+ return o.file.MimeType
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+ return o.file.ID
+}
+
+func getRemote(dir, fileName string) string {
+ if dir == "" {
+ return fileName
+ }
+ return dir + "/" + fileName
+}
+
+// Check the interfaces are satisfied
+var (
+ _ fs.Fs = (*Fs)(nil)
+ _ fs.Mover = (*Fs)(nil)
+ _ fs.DirMover = (*Fs)(nil)
+ _ fs.Copier = (*Fs)(nil)
+ _ fs.Purger = (*Fs)(nil)
+ _ fs.PublicLinker = (*Fs)(nil)
+ _ fs.Abouter = (*Fs)(nil)
+ _ fs.DirCacheFlusher = (*Fs)(nil)
+ _ dircache.DirCacher = (*Fs)(nil)
+ _ fs.Object = (*Object)(nil)
+ _ fs.MimeTyper = (*Object)(nil)
+ _ fs.IDer = (*Object)(nil)
+)
diff --git a/backend/dosya/dosya_api.go b/backend/dosya/dosya_api.go
new file mode 100644
index 000000000..33f93a65e
--- /dev/null
+++ b/backend/dosya/dosya_api.go
@@ -0,0 +1,880 @@
+package dosya
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/rclone/rclone/backend/dosya/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/readers"
+ "github.com/rclone/rclone/lib/rest"
+ "golang.org/x/sync/errgroup"
+)
+
+// sourceMTimeHeader is the request header the upload endpoints read to set a
+// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
+const sourceMTimeHeader = "x-dosya-source-mtime"
+
+// modTimeHeaders returns the ExtraHeaders that carry a source modification time
+// on an upload, or nil when there is nothing plausible to send. The server
+// clamps implausible values itself, but a zero time is never worth a header.
+func modTimeHeaders(modTime time.Time) map[string]string {
+ if modTime.IsZero() {
+ return nil
+ }
+ return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
+}
+
+// workspaceHintHeader names the workspace a by-id request is about. The API
+// maps an id to its workspace through an index the workspace fills about a
+// second after a write, so an id this client was handed moments ago can still
+// miss; the hint lets the server look for it in the named workspace instead.
+// It only locates: every route still checks membership against the workspace
+// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
+const workspaceHintHeader = "X-Dosya-Workspace"
+
+// byIDHeaders returns the ExtraHeaders every /api/files/:id and
+// /api/folders/:id call carries, or nil when no workspace is configured.
+func (f *Fs) byIDHeaders() map[string]string {
+ if f.opt.WorkspaceID == "" {
+ return nil
+ }
+ return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+ 429, // Too Many Requests
+ 500, // Internal Server Error
+ 502, // Bad Gateway
+ 503, // Service Unavailable
+ 504, // Gateway Timeout
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+ if fserrors.ContextError(ctx, &err) {
+ return false, err
+ }
+ retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
+ if retry {
+ if d, ok := retryAfter(resp); ok {
+ // The API answers 503 + Retry-After while a workspace is being
+ // moved between storage homes, and Cloudflare sends it with 429.
+ // The pacer's own backoff tops out at maxSleep, far short of that.
+ fs.Debugf(nil, "Server asked to retry after %v", d)
+ err = pacer.RetryAfterError(err, d)
+ }
+ }
+ return retry, err
+}
+
+// maxRetryAfter caps how long a single Retry-After is honoured for
+const maxRetryAfter = 5 * time.Minute
+
+// retryAfter reads a 429 or 503 response's Retry-After header, which the API
+// sends as whole seconds
+func retryAfter(resp *http.Response) (time.Duration, bool) {
+ if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
+ return 0, false
+ }
+ seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
+ if err != nil || seconds <= 0 {
+ return 0, false
+ }
+ return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
+}
+
+// errorHandler decodes a non-2xx response into an *api.Error so callers can
+// act on the machine-readable error_code rather than the message text.
+func errorHandler(resp *http.Response) error {
+ body, err := rest.ReadBody(resp)
+ if err != nil {
+ return fmt.Errorf("error reading error out of body: %w", err)
+ }
+ apiErr := &api.Error{
+ StatusCode: resp.StatusCode,
+ Status: resp.Status,
+ }
+ if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
+ // Not the API's JSON shape (e.g. an edge response that never
+ // reached the API) - keep the body text so it isn't lost.
+ apiErr.ErrorCode = ""
+ apiErr.Message = strings.TrimSpace(string(body))
+ }
+ return apiErr
+}
+
+// The workspace caps the number of in-flight uploads per user. The API
+// refuses an upload/init past that cap with a 400 carrying
+// error_code "concurrent_upload_limit", and documents it as an error to
+// wait on rather than give up on: slots free as uploads finish, and the
+// server marks sessions idle for 10 minutes as failed, so waiting up to
+// that long guarantees progress unless the workspace really is saturated.
+// A 400 is deliberately outside the pacer's retry codes, so the wait
+// lives in initUpload instead of shouldRetry.
+var (
+ concurrentUploadRetryDelay = 5 * time.Second
+ concurrentUploadWaitMax = 10 * time.Minute
+)
+
+// abortUploadTimeout bounds the abort of a session the client is giving up on,
+// which runs outside the transfer's context so a cancelled transfer still
+// sends it
+const abortUploadTimeout = 10 * time.Second
+
+// isConcurrentUploadLimit reports whether err is the API refusing an
+// upload because the workspace's concurrent upload limit is reached
+func isConcurrentUploadLimit(err error) bool {
+ var apiErr *api.Error
+ return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
+}
+
+// listPageSize is the largest page GET /api/files will serve
+const listPageSize = 500
+
+// listFilesAndFolders lists files and folders in a directory
+//
+// The API pages files (at most listPageSize per page) but returns every
+// folder on each page, so the files from all pages are gathered and the
+// folders are taken from the first page only. Pages are ordered by name,
+// which the API tie-breaks on id, so offsets are stable between requests.
+func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
+ var listing *api.ListResponse
+ for page := 1; ; page++ {
+ opts := rest.Opts{
+ Method: "GET",
+ Path: "/api/files",
+ Parameters: map[string][]string{
+ "workspace_id": {f.opt.WorkspaceID},
+ "per_page": {strconv.Itoa(listPageSize)},
+ "page": {strconv.Itoa(page)},
+ "sort": {"name"},
+ "dir": {"asc"},
+ },
+ }
+ if folderID != "" && folderID != rootID {
+ opts.Parameters["folder_id"] = []string{folderID}
+ }
+
+ var result api.ListResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't list files: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ if listing == nil {
+ listing = &result
+ } else {
+ listing.Files = append(listing.Files, result.Files...)
+ }
+ if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
+ return listing, nil
+ }
+ }
+}
+
+// createFolder creates a folder
+func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
+ request := api.CreateFolderRequest{
+ WorkspaceID: f.opt.WorkspaceID,
+ Name: name,
+ }
+ if parentID != "" && parentID != rootID {
+ request.ParentID = &parentID
+ }
+
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/api/folders",
+ }
+
+ var result api.CreateFolderResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't create folder: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
+
+// removeFolder deletes a folder
+//
+// ?permanent=1 trashes and purges a live folder in one request on today's
+// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
+// HTTP 409 "Folder is not empty" instead of purging a folder that still
+// holds a live file or a live child folder; that 409 is turned into
+// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
+// error, and it is never retried (409 is not one of retryErrorCodes). A big
+// subtree is purged in bounded passes (202, complete=false), so keep
+// calling until the purge is complete. An older server that ignores
+// permanent=1 only trashes on the first call (answering permanent=false),
+// so the loop's second pass sends the plain DELETE that purges it.
+func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
+ parameters := map[string][]string{"permanent": {"1"}}
+ if ifEmpty {
+ parameters["if_empty"] = []string{"1"}
+ }
+ opts := rest.Opts{
+ Method: "DELETE",
+ Path: "/api/folders/" + folderID,
+ Parameters: parameters,
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ for pass := 0; ; pass++ {
+ var result api.DeleteFolderResponse
+ var statusCode int
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ if resp != nil {
+ statusCode = resp.StatusCode
+ }
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ if ifEmpty && statusCode == http.StatusConflict {
+ return fs.ErrorDirectoryNotEmpty
+ }
+ if pass == 0 {
+ return fmt.Errorf("couldn't remove folder: %w", err)
+ }
+ return fmt.Errorf("couldn't permanently remove folder: %w", err)
+ }
+ if !result.OK {
+ return fmt.Errorf("API error: %s", result.Error)
+ }
+ if result.Permanent && result.Complete {
+ return nil
+ }
+ if pass >= maxFolderPurgePasses {
+ return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
+ }
+ }
+}
+
+// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
+const maxFolderPurgePasses = 1000
+
+// renameFolder renames a folder
+func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
+ request := api.RenameFolderRequest{Name: newName}
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/api/folders/" + folderID + "/rename",
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ var result api.Response
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("couldn't rename folder: %w", err)
+ }
+ if !result.OK {
+ return fmt.Errorf("API error: %s", result.Error)
+ }
+ return nil
+}
+
+// moveFolder moves a folder to a new parent
+func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
+ request := api.MoveFolderRequest{Name: newName}
+ if newParentID != "" && newParentID != rootID {
+ request.ParentID = &newParentID
+ }
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/api/folders/" + folderID + "/move",
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ var result api.MoveFolderResponse
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return false, fmt.Errorf("couldn't move folder: %w", err)
+ }
+ if !result.OK {
+ return false, fmt.Errorf("API error: %s", result.Error)
+ }
+ // Servers that predate the name field ignore it and answer without a
+ // name, leaving the caller to rename separately
+ return newName != "" && result.Name == newName, nil
+}
+
+// deleteFile deletes a file
+//
+// ?permanent=1 trashes and purges it in one request on today's API. An
+// older server that ignores the flag only soft-deletes it, answering
+// permanent=false, so a second DELETE on the now soft-deleted file purges
+// it from the database and R2, as it always has.
+func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
+ opts := rest.Opts{
+ Method: "DELETE",
+ Path: "/api/files/" + fileID,
+ Parameters: map[string][]string{"permanent": {"1"}},
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ var result api.DeleteFileResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("couldn't delete file: %w", err)
+ }
+ if !result.OK {
+ return fmt.Errorf("API error: %s", result.Error)
+ }
+
+ // Answered into its own value so a field the second response omits
+ // can't be read off the first.
+ if !result.Permanent {
+ var permanent api.DeleteFileResponse
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("couldn't permanently delete file: %w", err)
+ }
+ if !permanent.OK {
+ return fmt.Errorf("API error: %s", permanent.Error)
+ }
+ }
+ return nil
+}
+
+// renameFile renames a file
+func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
+ request := api.RenameFileRequest{Name: newName}
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/api/files/" + fileID + "/rename",
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ var result api.RenameFileResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("couldn't rename file: %w", err)
+ }
+ if !result.OK {
+ return fmt.Errorf("API error: %s", result.Error)
+ }
+ return nil
+}
+
+// moveFile moves a file to a new folder and, if newName is non-empty,
+// renames it in the same request
+func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
+ request := api.MoveFileRequest{}
+ if folderID != "" && folderID != rootID {
+ request.FolderID = &folderID
+ }
+ if newName != "" {
+ request.Name = &newName
+ }
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/api/files/" + fileID + "/move",
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ var result api.Response
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("couldn't move file: %w", err)
+ }
+ if !result.OK {
+ return fmt.Errorf("API error: %s", result.Error)
+ }
+ return nil
+}
+
+// copyFile copies a file to a folder
+func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
+ request := api.CopyFileRequest{Name: name}
+ if folderID != "" && folderID != rootID {
+ request.FolderID = &folderID
+ }
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/api/files/" + fileID + "/copy",
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ var result api.CopyFileResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't copy file: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
+
+// downloadFile downloads a file by its ID
+//
+// The download endpoint answers 302 with a presigned R2 URL in Location, so
+// this is two calls. Both go through the rest client, which is built on
+// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
+// --user-agent, --bind, --dump and the proxy settings apply to downloads as
+// they do to every other call, and errorHandler and shouldRetry decide the
+// errors and the retries uniformly.
+func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
+ // NoRedirect stops the client following the 302 with the API's
+ // Authorization header attached. A 302 is not a 2xx, so Call reports it
+ // as an error (having read and closed the body); the Location header
+ // is what actually matters, and finding it clears the error, as in
+ // backend/linkbox.
+ opts := rest.Opts{
+ Method: "GET",
+ Path: "/api/files/" + fileID + "/download",
+ ExtraHeaders: f.byIDHeaders(),
+ NoRedirect: true,
+ }
+ var downloadURL string
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.Call(ctx, &opts)
+ if resp != nil {
+ if location := resp.Header.Get("Location"); location != "" {
+ downloadURL, err = location, nil
+ }
+ }
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't get download URL: %w", err)
+ }
+ if downloadURL == "" {
+ return nil, errors.New("no download URL in redirect response")
+ }
+
+ // Fetch the object from the presigned URL. It carries its own
+ // credentials and R2 refuses a request that also has an Authorization
+ // header, so the client-level bearer token is removed by setting the
+ // header empty: the rest client skips a header with an empty value.
+ opts = rest.Opts{
+ Method: "GET",
+ RootURL: downloadURL,
+ Options: options,
+ ExtraHeaders: map[string]string{"Authorization": ""},
+ }
+ var resp *http.Response
+ err = f.pacer.Call(func() (bool, error) {
+ var err error
+ resp, err = f.rest.Call(ctx, &opts)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't download file: %w", err)
+ }
+ return resp.Body, nil
+}
+
+// uploadSmallFile uploads a file smaller than 10MB in a single request
+func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/api/upload/" + sessionID,
+ Body: in,
+ ContentLength: &size,
+ ContentType: "application/octet-stream",
+ ExtraHeaders: modTimeHeaders(modTime),
+ }
+
+ // The body is a stream which can't be rewound, so a low level retry
+ // would resend whatever is left of it - nothing - and the server
+ // would store that as an empty file. Don't retry here; rclone's high
+ // level retries redo the whole transfer instead.
+ var result api.UploadCompleteResponse
+ err := f.pacer.CallNoRetry(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't upload file: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
+
+// uploadPart uploads a single part of a multipart upload
+func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
+ size := int64(len(part))
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
+ ContentLength: &size,
+ ContentType: "application/octet-stream",
+ }
+
+ var result api.UploadPartResponse
+ err := f.pacer.Call(func() (bool, error) {
+ // A fresh reader per attempt so a retry resends the whole part
+ opts.Body = bytes.NewReader(part)
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
+
+// completeUpload finalizes a multipart upload
+func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/api/upload/" + sessionID + "/complete",
+ ExtraHeaders: modTimeHeaders(modTime),
+ }
+
+ var result api.UploadCompleteResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't complete upload: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
+
+// initUpload initializes a new upload session
+func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
+ request := api.UploadInitRequest{
+ WorkspaceID: f.opt.WorkspaceID,
+ FileName: fileName,
+ FileSize: fileSize,
+ MimeType: mimeType,
+ FileID: fileID,
+ }
+ if folderID != "" && folderID != rootID {
+ request.FolderID = &folderID
+ }
+
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/api/upload/init",
+ }
+
+ var result api.UploadInitResponse
+ deadline := time.Now().Add(concurrentUploadWaitMax)
+ for {
+ result = api.UploadInitResponse{}
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err == nil {
+ break
+ }
+ if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
+ return nil, fmt.Errorf("couldn't init upload: %w", err)
+ }
+ fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
+ select {
+ case <-ctx.Done():
+ return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
+ case <-time.After(concurrentUploadRetryDelay):
+ }
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
+
+// uploadParts reads the source part by part and uploads the parts.
+//
+// Part 1 goes on its own: the server creates the R2 multipart upload (and a
+// new file's id and key) when it receives the first part, so parts sent
+// before that has finished would each start an upload of their own. The
+// rest go out opt.UploadConcurrency at a time. A part is only read from the
+// source once a buffer is free, so at most that many parts are held in
+// memory, and a short source is still found by the reader before the part
+// it could not fill is sent.
+func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
+ readPart := func(buf []byte, partNum int) ([]byte, error) {
+ currentPartSize := partSize
+ if partNum == totalParts {
+ currentPartSize = size - partSize*int64(totalParts-1)
+ }
+ part := buf[:currentPartSize]
+ if _, err := io.ReadFull(in, part); err != nil {
+ if err == io.EOF || err == io.ErrUnexpectedEOF {
+ return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
+ }
+ return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
+ }
+ return part, nil
+ }
+
+ concurrency := max(f.opt.UploadConcurrency, 1)
+ buffers := make(chan []byte, concurrency)
+ buffers <- make([]byte, partSize)
+ allocated := 1
+
+ first := <-buffers
+ part, err := readPart(first, 1)
+ if err != nil {
+ return err
+ }
+ if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
+ return err
+ }
+ buffers <- first
+
+ // A failed read has to take the parts already in flight down with it,
+ // or they upload to completion while the error waits on Wait.
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ g, gCtx := errgroup.WithContext(ctx)
+ var readErr error
+ for partNum := 2; partNum <= totalParts; partNum++ {
+ // Take a free buffer, allocating one while under the limit.
+ var buf []byte
+ select {
+ case buf = <-buffers:
+ default:
+ if allocated < concurrency {
+ buf = make([]byte, partSize)
+ allocated++
+ } else {
+ select {
+ case buf = <-buffers:
+ case <-gCtx.Done():
+ }
+ }
+ }
+ if buf == nil {
+ break // a part failed; its error comes back from Wait
+ }
+ part, err := readPart(buf, partNum)
+ if err != nil {
+ readErr = err
+ cancel()
+ break
+ }
+ g.Go(func() error {
+ defer func() { buffers <- buf }()
+ _, err := f.uploadPart(gCtx, part, sessionID, partNum)
+ return err
+ })
+ }
+ err = g.Wait()
+ if readErr != nil {
+ return readErr
+ }
+ return err
+}
+
+// abortUpload retires an upload session the client is giving up on.
+//
+// Best effort: the server reclaims an abandoned session itself, so a failure
+// here - including an older server with no such route - costs the slot the
+// wait it would have cost anyway and is not worth failing an upload over.
+// Sent on a context of its own so a cancelled transfer still gives its slot
+// back rather than leaving it to the sweep.
+func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
+ defer cancel()
+ opts := rest.Opts{
+ Method: "DELETE",
+ Path: "/api/upload/" + sessionID,
+ NoResponse: true,
+ }
+ err := f.pacer.CallNoRetry(func() (bool, error) {
+ resp, err := f.rest.Call(ctx, &opts)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
+ }
+}
+
+// uploadFile handles the full upload flow (small or multipart)
+func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
+ // Both refusals live here so a new file and a new version of an existing
+ // one answer the same way: upload/init needs a byte count up front, and
+ // the server stores no zero-length object.
+ if size < 0 {
+ return nil, errors.New("can't upload files with unknown size")
+ }
+ if size == 0 {
+ return nil, fs.ErrorCantUploadEmptyFiles
+ }
+
+ leaf := f.opt.Enc.FromStandardName(path.Base(remote))
+
+ initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
+ if err != nil {
+ return nil, err
+ }
+ // A session abandoned here holds one of the workspace's upload slots
+ // until the server's sweep retires it ten minutes later, and rclone's
+ // high-level retry opens a fresh session rather than resuming this one -
+ // so without this, a few failed attempts use up every slot.
+ defer func() {
+ if err != nil {
+ f.abortUpload(ctx, initResp.SessionID)
+ }
+ }()
+
+ // Count what the source actually supplies, so a source which ends
+ // early is reported as an error rather than stored as a truncated
+ // file.
+ counter := readers.NewCountingReader(in)
+
+ // Small file upload (single PUT, no resumable)
+ if initResp.Resumable == nil {
+ resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
+ if err != nil {
+ return nil, err
+ }
+ if got := int64(counter.BytesRead()); got != size {
+ return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
+ }
+ return resp, nil
+ }
+
+ // Multipart upload. Each part is read fully into memory before it is
+ // sent, so a short source is caught before the part goes out and a
+ // part which has to be retried is resent with all of its bytes.
+ partSize := initResp.Resumable.PartSize
+ totalParts := initResp.Resumable.TotalParts
+ // Both bounds matter. partSize*(totalParts-1) >= size means a part would
+ // have nothing to carry; size > partSize*totalParts means the last part
+ // is bigger than partSize, and readPart would slice past its buffer and
+ // panic.
+ if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
+ return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
+ }
+ if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
+ return nil, err
+ }
+
+ return f.completeUpload(ctx, initResp.SessionID, modTime)
+}
+
+// createShareLink creates a public share link for a file
+func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
+ request := api.ShareLinkRequest{}
+ if expire < fs.DurationOff {
+ days := int(time.Duration(expire).Hours() / 24)
+ if days < 1 {
+ days = 1
+ }
+ request.ExpiresInDays = &days
+ }
+
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/api/files/" + fileID + "/share",
+ ExtraHeaders: f.byIDHeaders(),
+ }
+
+ var result api.ShareLinkResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't create share link: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
+
+// getWorkspaceInfo fetches workspace storage info
+// listWorkspaces lists the workspaces this credential can reach. An API key
+// pinned to a workspace sees exactly one: its own.
+func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
+ opts := rest.Opts{
+ Method: "GET",
+ Path: "/api/workspaces",
+ }
+
+ var result api.WorkspaceListResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't list workspaces: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return result.Workspaces, nil
+}
+
+func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
+ opts := rest.Opts{
+ Method: "GET",
+ Path: "/api/workspaces/" + f.opt.WorkspaceID,
+ }
+
+ var result api.WorkspaceInfoResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("couldn't get workspace info: %w", err)
+ }
+ if !result.OK {
+ return nil, fmt.Errorf("API error: %s", result.Error)
+ }
+ return &result, nil
+}
diff --git a/backend/dosya/dosya_compat_test.go b/backend/dosya/dosya_compat_test.go
new file mode 100644
index 000000000..ced911be7
--- /dev/null
+++ b/backend/dosya/dosya_compat_test.go
@@ -0,0 +1,235 @@
+package dosya
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/dircache"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestShouldRetryHonoursRetryAfter(t *testing.T) {
+ ctx := context.Background()
+ for _, tc := range []struct {
+ name string
+ status int
+ header string
+ want time.Duration
+ ok bool
+ }{
+ {"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
+ {"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
+ {"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
+ {"missing header", http.StatusServiceUnavailable, "", 0, false},
+ {"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
+ {"502 ignores the header", http.StatusBadGateway, "30", 0, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
+ if tc.header != "" {
+ resp.Header.Set("Retry-After", tc.header)
+ }
+ retry, err := shouldRetry(ctx, resp, errorHandler(resp))
+ assert.True(t, retry)
+ d, ok := pacer.IsRetryAfter(err)
+ assert.Equal(t, tc.ok, ok)
+ if tc.ok {
+ assert.Equal(t, tc.want, d)
+ }
+ })
+ }
+}
+
+func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
+ resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
+ resp.Header.Set("Retry-After", "30")
+ retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
+ assert.False(t, retry)
+ _, ok := pacer.IsRetryAfter(err)
+ assert.False(t, ok)
+}
+
+func TestRemoveFolderPurgesPermanently(t *testing.T) {
+ // a big subtree purges in bounded passes: 202 complete=false repeats
+ // the same request until complete=true
+ answers := []struct {
+ status int
+ body string
+ }{
+ {http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
+ {http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
+ }
+ calls := 0
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ require.Equal(t, "DELETE", r.Method)
+ require.Equal(t, "/api/folders/fld_1", r.URL.Path)
+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+ assert.Equal(t, "", r.URL.Query().Get("if_empty"))
+ require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
+ a := answers[calls]
+ calls++
+ return jsonResp(a.status, a.body), nil
+ }))
+
+ require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
+ assert.Equal(t, len(answers), calls)
+}
+
+func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
+ calls := 0
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ calls++
+ return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
+ }))
+
+ err := f.removeFolder(context.Background(), "fld_1", false)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "locked file")
+ assert.Equal(t, 1, calls)
+}
+
+func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
+ calls := 0
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ calls++
+ assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+ assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+ return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
+ }))
+
+ err := f.removeFolder(context.Background(), "fld_1", true)
+ assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
+ assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
+}
+
+func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
+ calls := 0
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ calls++
+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+ assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+ }))
+
+ require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
+ assert.Equal(t, 1, calls)
+}
+
+func TestRmdirMakesOneRequest(t *testing.T) {
+ calls := 0
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ calls++
+ assert.Equal(t, "DELETE", r.Method)
+ assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+ assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+ }))
+ f.dirCache = dircache.New("", rootID, f)
+ require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
+ f.dirCache.Put("sub", "fld_1")
+
+ require.NoError(t, f.Rmdir(context.Background(), "sub"))
+ assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
+}
+
+func TestRmdirMapsNonEmptyConflict(t *testing.T) {
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
+ }))
+ f.dirCache = dircache.New("", rootID, f)
+ require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
+ f.dirCache.Put("sub", "fld_1")
+
+ err := f.Rmdir(context.Background(), "sub")
+ assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
+}
+
+func TestDeleteFileSendsPermanentFlag(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ permanent bool
+ wantCalls int
+ }{
+ {"today's API purges in one call", true, 1},
+ {"an older server ignores the flag and still gets a second DELETE", false, 2},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ calls := 0
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ calls++
+ assert.Equal(t, "DELETE", r.Method)
+ assert.Equal(t, "/api/files/fil_1", r.URL.Path)
+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+ return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
+ }))
+
+ require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
+ assert.Equal(t, tc.wantCalls, calls)
+ })
+ }
+}
+
+// decodeBody reads a JSON request body into a map
+func decodeBody(t *testing.T, r *http.Request) map[string]any {
+ b, err := io.ReadAll(r.Body)
+ require.NoError(t, err)
+ var m map[string]any
+ require.NoError(t, json.Unmarshal(b, &m))
+ return m
+}
+
+func TestMoveFolderSendsName(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ newName string
+ respBody string
+ wantRenamed bool
+ }{
+ {"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
+ {"older server ignores name", "dst", `{"ok":true}`, false},
+ {"no rename requested", "", `{"ok":true,"name":"src"}`, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
+ body := decodeBody(t, r)
+ assert.Equal(t, "fld_parent", body["parent_id"])
+ if tc.newName == "" {
+ assert.NotContains(t, body, "name")
+ } else {
+ assert.Equal(t, tc.newName, body["name"])
+ }
+ return jsonResp(http.StatusOK, tc.respBody), nil
+ }))
+
+ renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
+ require.NoError(t, err)
+ assert.Equal(t, tc.wantRenamed, renamed)
+ })
+ }
+}
+
+func TestCopyFileSendsName(t *testing.T) {
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
+ body := decodeBody(t, r)
+ assert.Equal(t, "fld_dst", body["folder_id"])
+ assert.Equal(t, "final.txt", body["name"])
+ return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
+ }))
+
+ resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
+ require.NoError(t, err)
+ assert.Equal(t, "fil_2", resp.FileID)
+ assert.Equal(t, "final.txt", resp.Name)
+}
diff --git a/backend/dosya/dosya_hash_modtime_test.go b/backend/dosya/dosya_hash_modtime_test.go
new file mode 100644
index 000000000..4b79b7aed
--- /dev/null
+++ b/backend/dosya/dosya_hash_modtime_test.go
@@ -0,0 +1,171 @@
+package dosya
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/backend/dosya/api"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/lib/dircache"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
+ f := &Fs{}
+ assert.Equal(t, time.Second, f.Precision())
+ assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
+ // Exactly SHA-256, nothing else.
+ assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
+}
+
+func TestObjectHashReturnsContentHash(t *testing.T) {
+ o := &Object{file: api.FileItem{ContentHash: "abc123"}}
+
+ got, err := o.Hash(context.Background(), hash.SHA256)
+ require.NoError(t, err)
+ assert.Equal(t, "abc123", got)
+
+ // A multipart object with no server-side hash yields "" and no error,
+ // exactly as an S3 multipart object does.
+ empty := &Object{file: api.FileItem{ContentHash: ""}}
+ got, err = empty.Hash(context.Background(), hash.SHA256)
+ require.NoError(t, err)
+ assert.Equal(t, "", got)
+
+ // Any other algorithm is unsupported.
+ _, err = o.Hash(context.Background(), hash.MD5)
+ assert.ErrorIs(t, err, hash.ErrUnsupported)
+}
+
+func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
+ mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+ o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
+ assert.True(t, mt.Equal(o.ModTime(context.Background())))
+}
+
+func TestModTimeHeaders(t *testing.T) {
+ assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
+
+ mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
+ h := modTimeHeaders(mt)
+ require.NotNil(t, h)
+ assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
+}
+
+func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
+ resp := &api.UploadCompleteResponse{}
+ resp.File.ID = "fil_1"
+ resp.File.Name = "x.txt"
+ resp.File.SizeBytes = 10
+ resp.File.CreatedAt = 1_000
+ resp.File.ContentHash = "deadbeef"
+
+ mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
+ item := fileFromUpload(resp, mt)
+ assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
+ assert.Equal(t, "deadbeef", item.ContentHash)
+ assert.Equal(t, float64(1_000), item.CreatedAt)
+
+ // A zero modtime falls back to the create time.
+ item = fileFromUpload(resp, time.Time{})
+ assert.Equal(t, float64(1_000), item.UpdatedAt)
+}
+
+// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
+// fake transport and asserts the source-mtime header actually rides on the PUT.
+func TestUploadSendsSourceMTimeHeader(t *testing.T) {
+ var putMTime string
+ rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ _, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
+ }
+ switch {
+ case r.Method == "POST" && r.URL.Path == "/api/upload/init":
+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
+ case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
+ putMTime = r.Header.Get(sourceMTimeHeader)
+ return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
+ }
+ t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+ return nil, nil
+ })
+ f := newFakeFs(rt)
+
+ mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+ resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
+ require.NoError(t, err)
+ assert.Equal(t, "h", resp.File.ContentHash)
+ assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
+}
+
+// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
+// the completion request for a multipart upload (the server stores the modtime
+// there, not on the individual parts).
+func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
+ var completeMTime string
+ rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ _, _ = io.ReadAll(r.Body)
+ }
+ switch {
+ case r.URL.Path == "/api/upload/init":
+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
+ case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
+ return jsonResp(http.StatusOK, `{"ok":true}`), nil
+ case r.URL.Path == "/api/upload/upl_1/complete":
+ completeMTime = r.Header.Get(sourceMTimeHeader)
+ return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
+ }
+ t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+ return nil, nil
+ })
+ f := newFakeFs(rt)
+
+ mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+ _, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
+ require.NoError(t, err)
+ assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
+}
+
+// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
+// server-side Copy returns: it must report the SOURCE's modtime and hash (the
+// copy is byte-identical and the API preserves both), not the copy's wall
+// clock. Reading "now" here made rclone's in-memory object disagree with the
+// one read back from the remote and failed FsCopy's fingerprint check.
+func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
+ rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+ switch {
+ case r.Method == "GET" && r.URL.Path == "/api/files":
+ // NewObject's existence probe on the destination: nothing there.
+ return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
+ case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
+ return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
+ }
+ t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+ return nil, nil
+ })
+ f := newFakeFs(rt)
+ f.dirCache = dircache.New("", rootID, f)
+
+ mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
+ ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+ src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
+ ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
+ CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
+ }}
+
+ dst, err := f.Copy(context.Background(), src, "f-copy.txt")
+ require.NoError(t, err)
+
+ assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
+ assert.Equal(t, int64(100), dst.Size())
+ got, err := dst.Hash(context.Background(), hash.SHA256)
+ require.NoError(t, err)
+ assert.Equal(t, "srchash", got, "copy keeps the source hash")
+}
diff --git a/backend/dosya/dosya_internal_test.go b/backend/dosya/dosya_internal_test.go
new file mode 100644
index 000000000..33bd5382d
--- /dev/null
+++ b/backend/dosya/dosya_internal_test.go
@@ -0,0 +1,156 @@
+package dosya
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/backend/dosya/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/rest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
+
+func TestErrorHandlerDecodesAPIError(t *testing.T) {
+ resp := &http.Response{
+ StatusCode: http.StatusBadRequest,
+ Status: "400 Bad Request",
+ Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),
+ }
+ err := errorHandler(resp)
+ var apiErr *api.Error
+ require.ErrorAs(t, err, &apiErr)
+ assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
+ assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
+ assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
+ assert.Contains(t, err.Error(), "uploads in progress")
+ assert.Contains(t, err.Error(), "concurrent_upload_limit")
+}
+
+func TestErrorHandlerNonJSONBody(t *testing.T) {
+ // e.g. an edge (Cloudflare) response that never reached the API
+ resp := &http.Response{
+ StatusCode: http.StatusTooManyRequests,
+ Status: "429 Too Many Requests",
+ Body: io.NopCloser(strings.NewReader("error code: 1015\n")),
+ }
+ err := errorHandler(resp)
+ var apiErr *api.Error
+ require.ErrorAs(t, err, &apiErr)
+ assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
+ assert.Equal(t, "", apiErr.ErrorCode)
+ assert.Contains(t, err.Error(), "429")
+ assert.Contains(t, err.Error(), "error code: 1015")
+}
+
+func TestShouldRetryAPIErrors(t *testing.T) {
+ ctx := context.Background()
+
+ // The concurrent upload limit is handled by initUpload's slot wait,
+ // not by the pacer's low level retries.
+ limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
+ retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
+ assert.False(t, retry)
+ assert.Equal(t, limitErr, err)
+
+ retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
+ assert.True(t, retry)
+
+ retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
+ assert.False(t, retry)
+}
+
+// swapWaitTimings shortens the slot wait timings for a test and returns a
+// function which restores them
+func swapWaitTimings(delay, max time.Duration) func() {
+ oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
+ concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
+ return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
+}
+
+// newTestFs makes an Fs which talks to srv
+func newTestFs(srv *httptest.Server) *Fs {
+ ctx := context.Background()
+ f := &Fs{
+ name: "test",
+ opt: Options{WorkspaceID: "ws_test"},
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(
+ pacer.MinSleep(time.Millisecond),
+ pacer.MaxSleep(time.Millisecond),
+ pacer.DecayConstant(decayConstant),
+ pacer.AttackConstant(attackConstant),
+ )),
+ }
+ f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
+ return f
+}
+
+// limitServer answers the first n init calls with the concurrent upload
+// limit error and the rest with success
+func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ assert.Equal(t, "/api/upload/init", r.URL.Path)
+ w.Header().Set("Content-Type", "application/json")
+ if calls.Add(1) <= n {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = w.Write([]byte(concurrentLimitBody))
+ return
+ }
+ w.WriteHeader(http.StatusCreated)
+ _, _ = w.Write([]byte(`{"ok":true}`))
+ }))
+}
+
+func TestInitUploadWaitsForFreeSlot(t *testing.T) {
+ defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
+ var calls atomic.Int32
+ srv := limitServer(t, 2, &calls)
+ defer srv.Close()
+ f := newTestFs(srv)
+
+ resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
+ require.NoError(t, err)
+ assert.True(t, resp.OK)
+ assert.Equal(t, int32(3), calls.Load())
+}
+
+func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
+ defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
+ var calls atomic.Int32
+ srv := limitServer(t, 1<<30, &calls)
+ defer srv.Close()
+ f := newTestFs(srv)
+
+ _, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
+ require.Error(t, err)
+ var apiErr *api.Error
+ require.ErrorAs(t, err, &apiErr)
+ assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
+ assert.Contains(t, err.Error(), "couldn't init upload")
+ assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
+}
+
+func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
+ defer swapWaitTimings(time.Minute, time.Hour)()
+ var calls atomic.Int32
+ srv := limitServer(t, 1<<30, &calls)
+ defer srv.Close()
+ f := newTestFs(srv)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ start := time.Now()
+ _, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
+ require.Error(t, err)
+ assert.ErrorIs(t, err, context.DeadlineExceeded)
+ assert.Less(t, time.Since(start), 5*time.Second)
+}
diff --git a/backend/dosya/dosya_list_test.go b/backend/dosya/dosya_list_test.go
new file mode 100644
index 000000000..545b7d5d1
--- /dev/null
+++ b/backend/dosya/dosya_list_test.go
@@ -0,0 +1,68 @@
+package dosya
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// fakeListServer serves GET /api/files the way the API does: files paged at
+// per_page, every folder repeated on each page
+func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
+ return func(r *http.Request) (*http.Response, error) {
+ require.Equal(t, "/api/files", r.URL.Path)
+ q := r.URL.Query()
+ assert.Equal(t, "name", q.Get("sort"))
+ assert.Equal(t, "asc", q.Get("dir"))
+ page, _ := strconv.Atoi(q.Get("page"))
+ perPage, _ := strconv.Atoi(q.Get("per_page"))
+ require.Positive(t, page)
+ require.Positive(t, perPage)
+ *pages = append(*pages, page)
+
+ var files []string
+ for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
+ files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
+ }
+ totalPages := max(1, (totalFiles+perPage-1)/perPage)
+ body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
+ `"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
+ strings.Join(files, ","), page, perPage, totalFiles, totalPages)
+ return jsonResp(http.StatusOK, body), nil
+ }
+}
+
+func TestListFilesAndFoldersFollowsPages(t *testing.T) {
+ var pages []int
+ f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
+
+ listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
+ require.NoError(t, err)
+
+ assert.Equal(t, []int{1, 2, 3}, pages)
+ require.Len(t, listing.Files, 2*listPageSize+7)
+ assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
+ // folders repeat on every page and must not be duplicated
+ assert.Len(t, listing.Folders, 1)
+}
+
+func TestListFilesAndFoldersSinglePage(t *testing.T) {
+ for _, total := range []int{0, 1, listPageSize} {
+ t.Run(strconv.Itoa(total), func(t *testing.T) {
+ var pages []int
+ f := newFakeFs(fakeListServer(t, total, &pages))
+
+ listing, err := f.listFilesAndFolders(context.Background(), rootID)
+ require.NoError(t, err)
+
+ assert.Equal(t, []int{1}, pages)
+ assert.Len(t, listing.Files, total)
+ })
+ }
+}
diff --git a/backend/dosya/dosya_parallel_test.go b/backend/dosya/dosya_parallel_test.go
new file mode 100644
index 000000000..05b58436d
--- /dev/null
+++ b/backend/dosya/dosya_parallel_test.go
@@ -0,0 +1,153 @@
+package dosya
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// parallelUploadServer answers the multipart endpoints without holding a
+// lock across a request, so parts really do overlap, and records what the
+// ordering rules below are about: how many parts were in flight at once, and
+// whether any part started before part 1 had finished.
+//
+// Part 1 matters because the API creates the R2 multipart upload (and, for a
+// new file, its id and key) on the first part it sees. Parts that race ahead
+// of part 1 would each create their own upload, and only one of those would
+// be completed.
+type parallelUploadServer struct {
+ t *testing.T
+ resumable string
+ partDelay time.Duration
+ failPart int // answer this part with a 400, 0 for none
+ mu sync.Mutex
+ inFlight int
+ maxFlight int
+ part1Done bool
+ earlyParts []int // parts that started before part 1 finished
+ parts map[int]string
+ completes int
+ aborts int
+}
+
+func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
+ var body []byte
+ if r.Body != nil {
+ var err error
+ body, err = io.ReadAll(r.Body)
+ require.NoError(s.t, err)
+ }
+ path := r.URL.Path
+ switch {
+ case r.Method == "POST" && path == "/api/upload/init":
+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
+ case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
+ var n int
+ _, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
+ require.NoError(s.t, err)
+
+ s.mu.Lock()
+ if n != 1 && !s.part1Done {
+ s.earlyParts = append(s.earlyParts, n)
+ }
+ s.inFlight++
+ s.maxFlight = max(s.maxFlight, s.inFlight)
+ s.mu.Unlock()
+
+ time.Sleep(s.partDelay)
+
+ s.mu.Lock()
+ s.inFlight--
+ if n == 1 {
+ s.part1Done = true
+ }
+ s.parts[n] = string(body)
+ fail := n == s.failPart
+ s.mu.Unlock()
+
+ if fail {
+ return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
+ }
+ return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
+ case r.Method == "POST" && path == "/api/upload/upl_1/complete":
+ s.mu.Lock()
+ s.completes++
+ s.mu.Unlock()
+ return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
+ case r.Method == "DELETE" && path == "/api/upload/upl_1":
+ s.mu.Lock()
+ s.aborts++
+ s.mu.Unlock()
+ return jsonResp(http.StatusOK, `{"ok":true}`), nil
+ }
+ s.t.Fatalf("unexpected request %s %s", r.Method, path)
+ return nil, nil
+}
+
+func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
+ return &parallelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
+}
+
+const tenParts = `{"part_size":2,"total_parts":10}`
+
+func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
+ srv := newParallelServer(t, tenParts)
+ f := newFakeFs(rtFunc(srv.roundTrip))
+ f.opt.UploadConcurrency = 4
+ content := "aabbccddeeffgghhiijj"
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.NoError(t, err)
+
+ assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
+ assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
+ assert.Equal(t, 1, srv.completes)
+ for i := 1; i <= 10; i++ {
+ assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
+ }
+}
+
+func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
+ srv := newParallelServer(t, tenParts)
+ f := newFakeFs(rtFunc(srv.roundTrip))
+ f.opt.UploadConcurrency = 1
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.NoError(t, err)
+ assert.Equal(t, 1, srv.maxFlight)
+ assert.Equal(t, 1, srv.completes)
+}
+
+func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
+ srv := newParallelServer(t, tenParts)
+ srv.failPart = 6
+ f := newFakeFs(rtFunc(srv.roundTrip))
+ f.opt.UploadConcurrency = 4
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "part 6")
+ assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
+}
+
+func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
+ // 20 bytes declared, 15 supplied: the short read is found by the
+ // reader, whichever parts are already in flight, and nothing completes.
+ srv := newParallelServer(t, tenParts)
+ f := newFakeFs(rtFunc(srv.roundTrip))
+ f.opt.UploadConcurrency = 4
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.Error(t, err)
+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+ assert.Equal(t, 0, srv.completes)
+ assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
+}
diff --git a/backend/dosya/dosya_test.go b/backend/dosya/dosya_test.go
new file mode 100644
index 000000000..24a92cb8d
--- /dev/null
+++ b/backend/dosya/dosya_test.go
@@ -0,0 +1,15 @@
+// Test Dosya filesystem interface
+package dosya
+
+import (
+ "testing"
+
+ "github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+ fstests.Run(t, &fstests.Opt{
+ RemoteName: "TestDosya:",
+ })
+}
diff --git a/backend/dosya/dosya_upload_test.go b/backend/dosya/dosya_upload_test.go
new file mode 100644
index 000000000..2b43ee3d6
--- /dev/null
+++ b/backend/dosya/dosya_upload_test.go
@@ -0,0 +1,247 @@
+package dosya
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/rest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// rtFunc adapts a function into an http.RoundTripper so the upload path
+// can be driven without a network. Unlike a real server it always reads
+// the whole request body and answers, which is exactly the misbehaviour
+// (accepting a short body as a complete upload) these tests guard against.
+type rtFunc func(*http.Request) (*http.Response, error)
+
+func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func jsonResp(status int, body string) *http.Response {
+ return &http.Response{
+ StatusCode: status,
+ Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
+ Header: http.Header{"Content-Type": {"application/json"}},
+ Body: io.NopCloser(strings.NewReader(body)),
+ ContentLength: int64(len(body)),
+ }
+}
+
+func newFakeFs(rt http.RoundTripper) *Fs {
+ ctx := context.Background()
+ f := &Fs{
+ name: "test",
+ opt: Options{WorkspaceID: "ws_test"},
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(
+ pacer.MinSleep(time.Millisecond),
+ pacer.MaxSleep(time.Millisecond),
+ pacer.DecayConstant(decayConstant),
+ pacer.AttackConstant(attackConstant),
+ )),
+ }
+ f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
+ return f
+}
+
+// fakeUploadServer records what an upload sends and lets a test inject
+// failures per request
+type fakeUploadServer struct {
+ t *testing.T
+ mu sync.Mutex
+ resumable string // JSON for the init "resumable" field, "" for a single PUT
+ puts [][]byte // bodies of PUT /api/upload/<id>
+ parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
+ completes int
+ aborts int // DELETE /api/upload/<id>
+ failFirst map[string]bool // path -> answer the first request with a 502
+ seenBefore map[string]bool
+}
+
+func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
+ return &fakeUploadServer{
+ t: t,
+ resumable: resumable,
+ parts: map[int][][]byte{},
+ failFirst: map[string]bool{},
+ seenBefore: map[string]bool{},
+ }
+}
+
+func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ var body []byte
+ if r.Body != nil {
+ var err error
+ body, err = io.ReadAll(r.Body)
+ require.NoError(s.t, err)
+ }
+ path := r.URL.Path
+ // Bodies are recorded for every attempt, including the ones answered
+ // with a 502, so tests can see exactly what each attempt carried.
+ fail := s.failFirst[path] && !s.seenBefore[path]
+ s.seenBefore[path] = true
+ bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
+ switch {
+ case r.Method == "POST" && path == "/api/upload/init":
+ if s.resumable != "" {
+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
+ }
+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
+ case r.Method == "PUT" && path == "/api/upload/upl_1":
+ s.puts = append(s.puts, body)
+ if fail {
+ return bad, nil
+ }
+ return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
+ case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
+ var n int
+ _, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
+ require.NoError(s.t, err)
+ s.parts[n] = append(s.parts[n], body)
+ if fail {
+ return bad, nil
+ }
+ return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
+ case r.Method == "POST" && path == "/api/upload/upl_1/complete":
+ s.completes++
+ return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
+ case r.Method == "DELETE" && path == "/api/upload/upl_1":
+ s.aborts++
+ return jsonResp(http.StatusOK, `{"ok":true}`), nil
+ }
+ s.t.Fatalf("unexpected request %s %s", r.Method, path)
+ return nil, nil
+}
+
+func TestUploadSmallSendsWholeBody(t *testing.T) {
+ srv := newFakeUploadServer(t, "")
+ f := newFakeFs(rtFunc(srv.roundTrip))
+ content := strings.Repeat("a", 100)
+
+ resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.NoError(t, err)
+ assert.Equal(t, "fil_1", resp.File.ID)
+ require.Len(t, srv.puts, 1)
+ assert.Equal(t, content, string(srv.puts[0]))
+}
+
+func TestUploadSmallRejectsShortSource(t *testing.T) {
+ // The source declares 100 bytes but only supplies 60. A server which
+ // accepts that as a complete upload must not turn into a success.
+ srv := newFakeUploadServer(t, "")
+ f := newFakeFs(rtFunc(srv.roundTrip))
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.Error(t, err)
+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+ assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
+}
+
+func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
+ // A retried PUT would resend an already-consumed reader, i.e. an
+ // empty body, which the server would store as an empty file. The
+ // single PUT therefore must not be retried at the low level.
+ srv := newFakeUploadServer(t, "")
+ srv.failFirst["/api/upload/upl_1"] = true
+ f := newFakeFs(rtFunc(srv.roundTrip))
+
+ content := strings.Repeat("a", 100)
+ _, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.Error(t, err)
+ require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
+ assert.Equal(t, content, string(srv.puts[0]))
+}
+
+const threeParts = `{"part_size":4,"total_parts":3}`
+
+func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
+ srv := newFakeUploadServer(t, threeParts)
+ f := newFakeFs(rtFunc(srv.roundTrip))
+ content := "0123456789"
+
+ resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.NoError(t, err)
+ assert.Equal(t, "fil_1", resp.File.ID)
+ assert.Equal(t, 1, srv.completes)
+ require.Len(t, srv.parts, 3)
+ assert.Equal(t, "0123", string(srv.parts[1][0]))
+ assert.Equal(t, "4567", string(srv.parts[2][0]))
+ assert.Equal(t, "89", string(srv.parts[3][0]))
+}
+
+func TestUploadMultipartRejectsShortSource(t *testing.T) {
+ // 10 bytes declared, 8 supplied: the last part must never be sent
+ // and the upload must not be completed.
+ srv := newFakeUploadServer(t, threeParts)
+ f := newFakeFs(rtFunc(srv.roundTrip))
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.Error(t, err)
+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+ assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
+ assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
+ assert.Equal(t, 0, srv.completes, "upload must not be completed")
+}
+
+func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
+ // A part which fails transiently must be resent with all its bytes,
+ // not with whatever is left of a consumed reader.
+ srv := newFakeUploadServer(t, threeParts)
+ srv.failFirst["/api/upload/upl_1/part/2"] = true
+ f := newFakeFs(rtFunc(srv.roundTrip))
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+ require.NoError(t, err)
+ require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
+ assert.Equal(t, "4567", string(srv.parts[2][0]))
+ assert.Equal(t, "4567", string(srv.parts[2][1]))
+ assert.Equal(t, "89", string(srv.parts[3][0]))
+ assert.Equal(t, 1, srv.completes)
+}
+
+// An upload that fails after init has to hand its session back: the session
+// holds one of the workspace's upload slots for ten minutes otherwise, and
+// rclone's high-level retry opens a new one rather than resuming it.
+func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ resumable string
+ content string
+ size int64
+ }{
+ {"small file short source", "", "0123", 10},
+ {"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ srv := newFakeUploadServer(t, tc.resumable)
+ f := newFakeFs(rtFunc(srv.roundTrip))
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
+ "file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
+
+ require.Error(t, err)
+ assert.Equal(t, 1, srv.aborts)
+ })
+ }
+}
+
+// A successful upload must not abort anything
+func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
+ srv := newFakeUploadServer(t, "")
+ f := newFakeFs(rtFunc(srv.roundTrip))
+
+ _, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
+ "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+
+ require.NoError(t, err)
+ assert.Equal(t, 0, srv.aborts)
+}
diff --git a/backend/dosya/dosya_workspace_hint_test.go b/backend/dosya/dosya_workspace_hint_test.go
new file mode 100644
index 000000000..193f7b0c9
--- /dev/null
+++ b/backend/dosya/dosya_workspace_hint_test.go
@@ -0,0 +1,73 @@
+package dosya
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// Every by-id call names the workspace, so the API can find an id it minted
+// moments ago before its file_home index row lands.
+func TestByIDCallsSendWorkspaceHint(t *testing.T) {
+ seen := map[string]string{}
+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+ seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
+ switch r.URL.Path {
+ case "/api/folders/fld_1":
+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+ case "/api/files/fil_1":
+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
+ case "/api/folders/fld_1/move":
+ return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
+ default:
+ return jsonResp(http.StatusOK, `{"ok":true}`), nil
+ }
+ }))
+ ctx := context.Background()
+
+ require.NoError(t, f.removeFolder(ctx, "fld_1", false))
+ require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
+ _, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
+ require.NoError(t, err)
+ require.NoError(t, f.deleteFile(ctx, "fil_1"))
+ require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
+ require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
+ _, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
+ require.NoError(t, err)
+ _, err = f.createShareLink(ctx, "fil_1", 0)
+ require.NoError(t, err)
+
+ for call, header := range seen {
+ assert.Equal(t, "ws_test", header, call)
+ }
+ assert.Len(t, seen, 8)
+}
+
+// The download's API request carries the hint; the presigned R2 fetch must not.
+func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
+ var apiHint, blobHint string
+ var srv *httptest.Server
+ srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/api/files/fil_1/download" {
+ apiHint = r.Header.Get("X-Dosya-Workspace")
+ http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
+ return
+ }
+ blobHint = r.Header.Get("X-Dosya-Workspace")
+ _, _ = w.Write([]byte("bytes"))
+ }))
+ defer srv.Close()
+ f := newTestFs(srv)
+ f.opt.APIURL = srv.URL
+
+ rc, err := f.downloadFile(context.Background(), "fil_1", nil)
+ require.NoError(t, err)
+ _ = rc.Close()
+
+ assert.Equal(t, "ws_test", apiHint)
+ assert.Equal(t, "", blobHint)
+}
diff --git a/backend/dosya/dosya_workspace_test.go b/backend/dosya/dosya_workspace_test.go
new file mode 100644
index 000000000..7d04a09db
--- /dev/null
+++ b/backend/dosya/dosya_workspace_test.go
@@ -0,0 +1,52 @@
+package dosya
+
+import (
+ "context"
+ "net/http"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// fakeWorkspaceServer answers GET /api/workspaces with the given body
+func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
+ return func(r *http.Request) (*http.Response, error) {
+ require.Equal(t, "/api/workspaces", r.URL.Path)
+ *calls++
+ return jsonResp(http.StatusOK, body), nil
+ }
+}
+
+func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
+ calls := 0
+ f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
+ f.opt.WorkspaceID = ""
+
+ require.NoError(t, f.findWorkspace(context.Background()))
+ assert.Equal(t, "ws_only", f.opt.WorkspaceID)
+ assert.Equal(t, 1, calls)
+}
+
+func TestFindWorkspaceListsTheChoices(t *testing.T) {
+ calls := 0
+ f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
+ `{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
+ f.opt.WorkspaceID = ""
+
+ err := f.findWorkspace(context.Background())
+ require.Error(t, err)
+ // the ids have to be in the message: they are the only place the user
+ // can read them, which is the whole point of listing them
+ assert.Contains(t, err.Error(), "ws_one (One)")
+ assert.Contains(t, err.Error(), "ws_two (Two)")
+ assert.Empty(t, f.opt.WorkspaceID)
+}
+
+func TestFindWorkspaceWithNone(t *testing.T) {
+ calls := 0
+ f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
+ f.opt.WorkspaceID = ""
+
+ require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
+}
diff --git a/bin/make_manual.py b/bin/make_manual.py
index 2ba0380b1..5e514cbbe 100755
--- a/bin/make_manual.py
+++ b/bin/make_manual.py
@@ -44,6 +44,7 @@ docs = [
"compress.md",
"combine.md",
"doi.md",
+ "dosya.md",
"drime.md",
"dropbox.md",
"filefabric.md",
diff --git a/docs/content/_index.md b/docs/content/_index.md
index 0e357f221..56373921e 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -128,6 +128,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Cubbit DS3" home="https://cubbit.io/ds3-cloud" config="/s3/#Cubbit" >}}
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
+{{< provider name="dosya.dev" home="https://dosya.dev/" config="/dosya/" >}}
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
{{< provider name="Drime" home="https://www.drime.cloud/" config="/drime/" >}}
{{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
diff --git a/docs/content/docs.md b/docs/content/docs.md
index e06a54aad..e0cd0b74b 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -43,6 +43,7 @@ See the following for detailed instructions for
- [Crypt](/crypt/) - to encrypt other remotes
- [DigitalOcean Spaces](/s3/#digitalocean-spaces)
- [Digi Storage](/koofr/#digi-storage)
+- [dosya.dev](/dosya/)
- [Drime](/drime/)
- [Dropbox](/dropbox/)
- [Enterprise File Fabric](/filefabric/)
diff --git a/docs/content/dosya.md b/docs/content/dosya.md
new file mode 100644
index 000000000..5f2c399de
--- /dev/null
+++ b/docs/content/dosya.md
@@ -0,0 +1,286 @@
+---
+title: "dosya.dev"
+description: "Rclone docs for dosya.dev"
+versionIntroduced: "v1.76"
+---
+
+# dosya.dev
+
+[dosya.dev](https://dosya.dev) is a cloud file storage, sharing, and
+synchronization platform with workspace-based multi-tenancy, file
+versioning, and team collaboration features.
+
+## Configuration
+
+Here is an example of how to make a remote called `remote`. First run:
+
+```console
+rclone config
+```
+
+This will guide you through an interactive setup process:
+
+```text
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+
+Enter name for new remote.
+name> remote
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+XX / dosya.dev
+ \ (dosya)
+Storage> dosya
+
+Option api_key.
+Your API Key, get it from https://dosya.dev/profile under "API keys".
+Enter a value.
+api_key> dos_YOURAPIKEY
+
+Option workspace_id.
+Your workspace ID.
+
+Leave this blank to use the one workspace the API key can reach, which
+is what a key restricted to a single workspace does. If the key can
+reach several, rclone will refuse and list their IDs for you to choose
+from.
+Enter a value. Press Enter to leave empty.
+workspace_id>
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: dosya
+- api_key: dos_YOURAPIKEY
+Keep this "remote" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Once configured you can then use `rclone` like this (replace `remote`
+with the name you gave your remote):
+
+List directories and files in the top level of your dosya workspace
+
+```console
+rclone lsf remote:
+```
+
+To copy a local directory to a dosya directory called backup
+
+```console
+rclone copy /home/source remote:backup
+```
+
+The dosya backend uploads large files in parts and stores them as single
+files that remain accessible through the web app and other clients. If you
+previously used a WebDAV remote with a chunker overlay for large uploads,
+you can use this backend without the overlay.
+
+### Getting your API key
+
+1. Log in to [dosya.dev](https://dosya.dev)
+2. Go to [Profile](https://dosya.dev/profile) > **API keys**
+3. Create a new API key
+4. Copy the key (starts with `dos_`)
+
+A key can be restricted to a single workspace when you create it, which
+is the simplest thing to do for rclone: `workspace_id` can then be left
+out of the config entirely.
+
+### Workspaces
+
+Leave `workspace_id` blank and rclone asks which workspaces the key can
+reach. A key restricted to one workspace, and an account that has only
+one, both resolve without any configuration. If the key can reach
+several, rclone refuses and lists their IDs so you can set the one you
+want:
+
+```text
+this API key can reach 2 workspaces, so workspace_id must be set to one of:
+ ws_AAAAAAAA (Work)
+ ws_BBBBBBBB (Personal)
+```
+
+To access several workspaces at once, create a separate remote for each:
+
+```console
+rclone config create work dosya api_key=dos_XXX workspace_id=ws_WORK
+rclone config create personal dosya api_key=dos_XXX workspace_id=ws_PERSONAL
+```
+
+Then use them independently:
+
+```console
+rclone ls work:
+rclone copy work:docs/ personal:backup/
+```
+
+### Modification times and hashes
+
+dosya.dev supports modification times with a resolution of 1 second. The
+modification time is sent with the upload, so a copied or synced file keeps
+its original timestamp. Modification times cannot be changed after upload
+without re-uploading the file.
+
+dosya.dev supports SHA-256 checksums. The server computes the hash of a
+single-shot (small file) upload from the bytes it receives, so `rclone check`
+and hash-based sync work without downloading. Large files uploaded in multiple
+parts have no server-side hash (the parts are never re-read to hash the whole
+object), the same limitation the S3 backend has for multipart objects; those
+objects report an empty hash.
+
+### Restricted filename characters
+
+In addition to the [default restricted characters set](/overview/#restricted-characters)
+the following characters are also replaced:
+
+| Character | Value | Replacement |
+| --------- |:-----:|:-----------:|
+| " | 0x22 | " |
+| ' | 0x27 | ' |
+| < | 0x3C | < |
+| > | 0x3E | > |
+| \ | 0x5C | \ |
+
+Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
+as they can't be used in JSON strings.
+
+### Public Links
+
+dosya.dev supports `rclone link` to create public share links to
+files. You can use the `--expire` flag to specify how long the link
+should be valid.
+
+```console
+rclone link remote:path/to/file.txt
+rclone link --expire 7d remote:path/to/file.txt
+```
+
+<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/dosya/dosya.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
+### Standard options
+
+Here are the Standard options specific to dosya (dosya.dev).
+
+#### --dosya-api-key
+
+Your API Key, get it from https://dosya.dev/profile under "API keys".
+
+Properties:
+
+- Config: api_key
+- Env Var: RCLONE_DOSYA_API_KEY
+- Type: string
+- Required: true
+
+#### --dosya-workspace-id
+
+Your workspace ID.
+
+Leave this blank to use the one workspace the API key can reach, which
+is what a key restricted to a single workspace does. If the key can
+reach several, rclone will refuse and list their IDs for you to choose
+from.
+
+Properties:
+
+- Config: workspace_id
+- Env Var: RCLONE_DOSYA_WORKSPACE_ID
+- Type: string
+- Required: false
+
+### Advanced options
+
+Here are the Advanced options specific to dosya (dosya.dev).
+
+#### --dosya-api-url
+
+API base URL.
+
+Properties:
+
+- Config: api_url
+- Env Var: RCLONE_DOSYA_API_URL
+- Type: string
+- Default: "https://api.dosya.dev"
+
+#### --dosya-upload-concurrency
+
+Concurrency for multipart uploads.
+
+This is the number of parts of the same file that are uploaded
+concurrently. The first part is always uploaded on its own, because
+the server creates the upload when it receives it.
+
+Each part in flight is held in memory, so memory use grows with
+this value times the part size the server chooses.
+
+If you are uploading large files over high-speed links and these
+uploads do not fully utilize your bandwidth, then increasing this
+may help to speed up the transfers.
+
+Properties:
+
+- Config: upload_concurrency
+- Env Var: RCLONE_DOSYA_UPLOAD_CONCURRENCY
+- Type: int
+- Default: 4
+
+#### --dosya-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](/overview/#encoding) for more info.
+
+Properties:
+
+- Config: encoding
+- Env Var: RCLONE_DOSYA_ENCODING
+- Type: Encoding
+- Default: Slash,LtGt,DoubleQuote,SingleQuote,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot
+
+#### --dosya-description
+
+Description of the remote.
+
+Properties:
+
+- Config: description
+- Env Var: RCLONE_DOSYA_DESCRIPTION
+- Type: string
+- Required: false
+
+<!-- autogenerated options stop -->
+
+## Limitations
+
+dosya.dev only supports filenames up to 255 characters in length.
+
+Empty files (0 bytes) cannot be uploaded to dosya.dev.
+
+Upload requires knowing the file size in advance, so streaming
+uploads of unknown-size files are not supported.
+
+Each workspace limits the number of uploads a user can have in flight
+at once (5 by default, configurable in the workspace settings). When
+`--transfers` is set higher than that limit, rclone waits for a free
+upload slot rather than failing the transfer, retrying every 5 seconds
+for up to 10 minutes.
+
+### Deleting files and directories
+
+rclone deletes files and directories permanently, in a single request
+(`DELETE ?permanent=1`), bypassing the workspace trash. Removing a
+directory also asks the server whether it is empty (`?if_empty=1`)
+rather than listing it first.
diff --git a/docs/data/backends/dosya.yaml b/docs/data/backends/dosya.yaml
new file mode 100644
index 000000000..c12c337c1
--- /dev/null
+++ b/docs/data/backends/dosya.yaml
@@ -0,0 +1,26 @@
+backend: dosya
+name: dosya.dev
+tier: Tier 1
+maintainers: External
+features_score: 5
+integration_tests: Passing
+data_integrity: Hash
+performance: Medium
+adoption: New
+docs: Full
+security: High
+virtual: false
+remote: 'TestDosya:'
+features:
+- About
+- CanHaveEmptyDirectories
+- Copy
+- DirCacheFlush
+- DirMove
+- Move
+- PublicLink
+- Purge
+- ReadMimeType
+hashes:
+- sha256
+precision: 1000000000
diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html
index b7f80d285..cff6fa02d 100644
--- a/docs/layouts/chrome/navbar.html
+++ b/docs/layouts/chrome/navbar.html
@@ -70,6 +70,7 @@
<a class="dropdown-item" href="/crypt/">Crypt (encrypts the others)</a>
<span class="dropdown-letter-heading">D &ndash; F</span>
<a class="dropdown-item" href="/koofr/#digi-storage">Digi Storage</a>
+ <a class="dropdown-item" href="/dosya/">dosya.dev</a>
<a class="dropdown-item" href="/drime/">Drime</a>
<a class="dropdown-item" href="/dropbox/">Dropbox</a>
<a class="dropdown-item" href="/filefabric/">Enterprise File Fabric</a>
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index 81425dcb6..3cb9af961 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -723,6 +723,14 @@ backends:
listretries: 5
ignore:
- TestRWFileHandleWriteNoWrite
+ - backend: "dosya"
+ remote: "TestDosya:"
+ # Every upload is a metadata round trip of several seconds regardless
+ # of size, so fs/sync (TestTransform alone is 11 x ~3.5 min) needs
+ # well over the default 1h budget.
+ extratime: 3.0
+ ignore:
+ - TestRWFileHandleWriteNoWrite
- backend: "drime"
remote: "TestDrime:"
ignoretests: