"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
Add new backend dosya for dosya.dev cloud storage
Kaya408cc55171b6
diff --git a/README.md b/README.mdindex e4456493f..f1eab0da4 100644--- a/README.md+++ b/README.md@@ -39,6 +39,7 @@ directories to and from different cloud storage providers.- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)+- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)- Drime [:page_facing_up:](https://rclone.org/s3/#drime)- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)diff --git a/backend/all/all.go b/backend/all/all.goindex 37c8f56e9..0d8bed8c8 100644--- a/backend/all/all.go+++ b/backend/all/all.go@@ -16,6 +16,7 @@ import (_ "github.com/rclone/rclone/backend/compress"_ "github.com/rclone/rclone/backend/crypt"_ "github.com/rclone/rclone/backend/doi"+ _ "github.com/rclone/rclone/backend/dosya"_ "github.com/rclone/rclone/backend/drime"_ "github.com/rclone/rclone/backend/drive"_ "github.com/rclone/rclone/backend/dropbox"diff --git a/backend/dosya/api/types.go b/backend/dosya/api/types.gonew file mode 100644index 000000000..5d0cb89a7--- /dev/null+++ b/backend/dosya/api/types.go@@ -0,0 +1,259 @@+// Package api has type definitions for dosya.dev+package api++import "fmt"++// Response is the base response wrapper+type Response struct {+ OK bool `json:"ok"`+ Error string `json:"error,omitempty"`+}++// Error is an error response from the API.+//+// StatusCode and Status are filled in from the HTTP response. The API+// sets ErrorCode on the errors a client is expected to act on rather+// than just report, e.g. "concurrent_upload_limit".+type Error struct {+ StatusCode int `json:"-"`+ Status string `json:"-"`+ OK bool `json:"ok"`+ Message string `json:"error"`+ ErrorCode string `json:"error_code,omitempty"`+ MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`+}++// Error satisfies the error interface+func (e *Error) Error() string {+ out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)+ if e.Message != "" {+ out += ": " + e.Message+ }+ if e.ErrorCode != "" {+ out += " (" + e.ErrorCode + ")"+ }+ return out+}++// FileItem represents a file from the dosya API+type FileItem struct {+ ID string `json:"id"`+ Name string `json:"name"`+ SizeBytes int64 `json:"size_bytes"`+ MimeType string `json:"mime_type"`+ Extension string `json:"extension"`+ Region string `json:"region"`+ CreatedAt float64 `json:"created_at"`+ UpdatedAt float64 `json:"updated_at"`+ FolderID *string `json:"folder_id"`+ IsSynced int `json:"is_synced"`+ // ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when+ // the server has none (e.g. a file uploaded as an R2 multipart, whose parts+ // are never re-read to hash the whole object - the same limitation S3 has).+ ContentHash string `json:"content_hash"`+}++// FolderItem represents a folder from the dosya API+type FolderItem struct {+ ID string `json:"id"`+ Name string `json:"name"`+ ParentID *string `json:"parent_id"`+ CreatedAt float64 `json:"created_at"`+ FileCount int `json:"file_count"`+ IsSynced int `json:"is_synced"`+}++// ListResponse is the response from GET /api/files+type ListResponse struct {+ Response+ Files []FileItem `json:"files"`+ Folders []FolderItem `json:"folders"`+ Pagination Pagination `json:"pagination"`+}++// Pagination describes which page of files a ListResponse holds+type Pagination struct {+ Page int `json:"page"`+ PerPage int `json:"per_page"`+ TotalFiles int `json:"total_files"`+ TotalPages int `json:"total_pages"`+}++// CreateFolderRequest is the request body for POST /api/folders+type CreateFolderRequest struct {+ WorkspaceID string `json:"workspace_id"`+ ParentID *string `json:"parent_id"`+ Name string `json:"name"`+}++// CreateFolderResponse is the response from POST /api/folders+type CreateFolderResponse struct {+ Response+ Folder struct {+ ID string `json:"id"`+ Name string `json:"name"`+ ParentID *string `json:"parent_id"`+ Workspace string `json:"workspace_id"`+ } `json:"folder"`+}++// DeleteFolderResponse is the response from DELETE /api/folders/:id+type DeleteFolderResponse struct {+ Response+ Permanent bool `json:"permanent"`+ Complete bool `json:"complete"`+ Remaining int `json:"remaining"`+ FilesAffected int `json:"files_affected"`+ FoldersRemoved int `json:"folders_removed"`+}++// RenameFolderRequest is the request body for PUT /api/folders/:id/rename+type RenameFolderRequest struct {+ Name string `json:"name"`+}++// MoveFolderRequest is the request body for PUT /api/folders/:id/move+type MoveFolderRequest struct {+ ParentID *string `json:"parent_id"`+ Name string `json:"name,omitempty"`+}++// MoveFolderResponse is the response from PUT /api/folders/:id/move+type MoveFolderResponse struct {+ Response+ Name string `json:"name"`+}++// UploadInitRequest is the request body for POST /api/upload/init+type UploadInitRequest struct {+ WorkspaceID string `json:"workspace_id"`+ FileName string `json:"file_name"`+ FileSize int64 `json:"file_size"`+ MimeType string `json:"mime_type"`+ Region string `json:"region,omitempty"`+ FolderID *string `json:"folder_id"`+ FileID *string `json:"file_id,omitempty"`+}++// UploadInitResponse is the response from POST /api/upload/init+type UploadInitResponse struct {+ Response+ SessionID string `json:"session_id"`+ UploadURL string `json:"upload_url"`+ WorkspaceID string `json:"workspace_id"`+ FileName string `json:"file_name"`+ FileSize int64 `json:"file_size"`+ MimeType string `json:"mime_type"`+ Extension string `json:"extension"`+ Region string `json:"region"`+ Resumable *struct {+ PartSize int64 `json:"part_size"`+ TotalParts int `json:"total_parts"`+ PartUploadURL string `json:"part_upload_url"`+ CompleteURL string `json:"complete_url"`+ StatusURL string `json:"status_url"`+ } `json:"resumable"`+}++// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete+type UploadCompleteResponse struct {+ Response+ File struct {+ ID string `json:"id"`+ Name string `json:"name"`+ SizeBytes int64 `json:"size_bytes"`+ MimeType string `json:"mime_type"`+ Extension string `json:"extension"`+ Region string `json:"region"`+ Version int `json:"version"`+ CreatedAt float64 `json:"created_at"`+ // SHA-256 of the uploaded bytes for a single-shot (small-file) upload;+ // "" for a multipart upload, which the server does not hash.+ ContentHash string `json:"content_hash"`+ } `json:"file"`+}++// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber+type UploadPartResponse struct {+ Response+ PartNumber int `json:"part_number"`+ ETag string `json:"etag"`+ BytesUploaded int64 `json:"bytes_uploaded"`+ PartsCompleted int `json:"parts_completed"`+ TotalParts int `json:"total_parts"`+}++// DeleteFileResponse is the response from DELETE /api/files/:id+type DeleteFileResponse struct {+ Response+ Permanent bool `json:"permanent"`+}++// RenameFileRequest is the request body for PUT /api/files/:id/rename+type RenameFileRequest struct {+ Name string `json:"name"`+}++// RenameFileResponse is the response from PUT /api/files/:id/rename+type RenameFileResponse struct {+ Response+ Name string `json:"name"`+}++// MoveFileRequest is the request body for PUT /api/files/:id/move+type MoveFileRequest struct {+ FolderID *string `json:"folder_id"`+ Name *string `json:"name,omitempty"`+}++// CopyFileRequest is the request body for POST /api/files/:id/copy+type CopyFileRequest struct {+ FolderID *string `json:"folder_id"`+ Name string `json:"name,omitempty"`+}++// CopyFileResponse is the response from POST /api/files/:id/copy+type CopyFileResponse struct {+ Response+ FileID string `json:"file_id"`+ Name string `json:"name"`+}++// ShareLinkRequest is the request body for POST /api/files/:id/share+type ShareLinkRequest struct {+ ExpiresInDays *int `json:"expires_in_days,omitempty"`+}++// ShareLinkResponse is the response from POST /api/files/:id/share+type ShareLinkResponse struct {+ Response+ Link struct {+ ID string `json:"id"`+ Token string `json:"token"`+ URL string `json:"url"`+ ExpiresAt *int64 `json:"expires_at"`+ CreatedAt float64 `json:"created_at"`+ } `json:"link"`+}++// Workspace is one entry of the workspace list+type Workspace struct {+ ID string `json:"id"`+ Name string `json:"name"`+}++// WorkspaceListResponse is the response from GET /api/workspaces+type WorkspaceListResponse struct {+ Response+ Workspaces []Workspace `json:"workspaces"`+}++// WorkspaceInfoResponse is the response from GET /api/workspaces/:id+type WorkspaceInfoResponse struct {+ Response+ Storage struct {+ Used int64 `json:"used"`+ Total int64 `json:"total"`+ Free int64 `json:"free"`+ } `json:"storage"`+}diff --git a/backend/dosya/dosya.go b/backend/dosya/dosya.gonew file mode 100644index 000000000..4256d4674--- /dev/null+++ b/backend/dosya/dosya.go@@ -0,0 +1,772 @@+// Package dosya provides an interface to the dosya.dev cloud storage system.+package dosya++import (+ "context"+ "errors"+ "fmt"+ "io"+ "strings"+ "time"++ "github.com/rclone/rclone/backend/dosya/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/lib/dircache"+ "github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/random"+ "github.com/rclone/rclone/lib/rest"+)++const (+ rootID = "root"+ defaultBaseURL = "https://api.dosya.dev"+ minSleep = 100 * time.Millisecond+ maxSleep = 2 * time.Second+ decayConstant = 2+ attackConstant = 0+)++func init() {+ fs.Register(&fs.RegInfo{+ Name: "dosya",+ Description: "dosya.dev",+ NewFs: NewFs,+ Options: []fs.Option{{+ Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",+ Name: "api_key",+ Sensitive: true,+ Required: true,+ }, {+ Name: "workspace_id",+ Help: `Your workspace ID.++Leave this blank to use the one workspace the API key can reach, which+is what a key restricted to a single workspace does. If the key can+reach several, rclone will refuse and list their IDs for you to choose+from.`,+ }, {+ Help: "API base URL.",+ Name: "api_url",+ Default: defaultBaseURL,+ Advanced: true,+ }, {+ Name: "upload_concurrency",+ Help: `Concurrency for multipart uploads.++This is the number of parts of the same file that are uploaded+concurrently. The first part is always uploaded on its own, because+the server creates the upload when it receives it.++Each part in flight is held in memory, so memory use grows with+this value times the part size the server chooses.++If you are uploading large files over high-speed links and these+uploads do not fully utilize your bandwidth, then increasing this+may help to speed up the transfers.`,+ Default: 4,+ Advanced: true,+ }, {+ Name: config.ConfigEncoding,+ Help: config.ConfigEncodingHelp,+ Advanced: true,+ Default: (encoder.Display |+ encoder.EncodeBackSlash |+ encoder.EncodeSingleQuote |+ encoder.EncodeDoubleQuote |+ encoder.EncodeLtGt |+ encoder.EncodeCtl |+ encoder.EncodeLeftSpace |+ encoder.EncodeRightSpace |+ encoder.EncodeInvalidUtf8),+ }},+ })+}++// Options defines the configuration for this backend+type Options struct {+ APIKey string `config:"api_key"`+ WorkspaceID string `config:"workspace_id"`+ APIURL string `config:"api_url"`+ UploadConcurrency int `config:"upload_concurrency"`+ Enc encoder.MultiEncoder `config:"encoding"`+}++// Fs represents a remote dosya.dev filesystem+type Fs struct {+ name string+ root string+ opt Options+ features *fs.Features+ rest *rest.Client+ pacer *fs.Pacer+ dirCache *dircache.DirCache+}++// Object represents a dosya.dev file+type Object struct {+ fs *Fs+ remote string+ file api.FileItem+}++// findWorkspace fills in the workspace ID when the config leaves it blank.+//+// The ID is not shown anywhere in the dosya.dev web interface, so requiring+// it meant asking people to find it in a developer console. A key restricted+// to one workspace can only mean that one, and an account with a single+// workspace has only one possible answer; anything else genuinely has to be+// chosen, so list the choices in the error.+func (f *Fs) findWorkspace(ctx context.Context) error {+ workspaces, err := f.listWorkspaces(ctx)+ if err != nil {+ return err+ }+ switch len(workspaces) {+ case 0:+ return errors.New("no workspaces found for this API key")+ case 1:+ f.opt.WorkspaceID = workspaces[0].ID+ fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)+ return nil+ }+ var choices strings.Builder+ for _, w := range workspaces {+ _, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)+ }+ return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",+ len(workspaces), choices.String())+}++// FindLeaf finds a directory of name leaf in the folder with ID pathID+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {+ listing, err := f.listFilesAndFolders(ctx, pathID)+ if err != nil {+ return "", false, err+ }+ for _, folder := range listing.Folders {+ if f.opt.Enc.ToStandardName(folder.Name) == leaf {+ return folder.ID, true, nil+ }+ }+ return "", false, nil+}++// CreateDir makes a directory with pathID as parent and name leaf+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {+ name := f.opt.Enc.FromStandardName(leaf)+ resp, err := f.createFolder(ctx, name, pathID)+ if err != nil {+ return "", err+ }+ return resp.Folder.ID, nil+}++// Name of the remote (as passed into NewFs)+func (f *Fs) Name() string {+ return f.name+}++// Root of the remote (as passed into NewFs)+func (f *Fs) Root() string {+ return f.root+}++// String returns a description of the FS+func (f *Fs) String() string {+ return fmt.Sprintf("dosya.dev root '%s'", f.root)+}++// Precision of the ModTimes in this Fs+func (f *Fs) Precision() time.Duration {+ // Modification times are stored and returned to the second (the API+ // keeps them as Unix seconds, sent on upload via the source-mtime header).+ return time.Second+}++// Hashes returns the supported hash types of the filesystem+func (f *Fs) Hashes() hash.Set {+ // The server computes and stores the SHA-256 of a single-shot upload's+ // bytes; multipart uploads have none (see Object.Hash).+ return hash.NewHashSet(hash.SHA256)+}++// Features returns the optional features of this Fs+func (f *Fs) Features() *fs.Features {+ return f.features+}++// NewFs makes a new Fs object from the path+func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {+ opt := new(Options)+ err := configstruct.Set(config, opt)+ if err != nil {+ return nil, err+ }++ root = strings.Trim(root, "/")++ f := &Fs{+ name: name,+ root: root,+ opt: *opt,+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),+ }++ // No DuplicateFiles: an upload that names no existing file but whose+ // (workspace, folder, name) is already taken becomes a new VERSION of+ // that file rather than a second row, so a name is unique within a+ // folder and the same upload twice returns the same file id.+ f.features = (&fs.Features{+ CaseInsensitive: false,+ CanHaveEmptyDirectories: true,+ ReadMimeType: true,+ }).Fill(ctx, f)++ client := fshttp.NewClient(ctx)+ f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)+ f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)++ if f.opt.WorkspaceID == "" {+ err = f.findWorkspace(ctx)+ if err != nil {+ return nil, err+ }+ }++ f.dirCache = dircache.New(root, rootID, f)++ // Find the current root+ err = f.dirCache.FindRoot(ctx, false)+ if err != nil {+ // Assume it is a file+ newRoot, remote := dircache.SplitPath(root)+ tempF := *f+ tempF.dirCache = dircache.New(newRoot, rootID, &tempF)+ tempF.root = newRoot+ // Make new Fs which is the parent+ err = tempF.dirCache.FindRoot(ctx, false)+ if err != nil {+ // No root so return old f+ return f, nil+ }+ _, err := tempF.NewObject(ctx, remote)+ if err != nil {+ if err == fs.ErrorObjectNotFound {+ // File doesn't exist so return old f+ return f, nil+ }+ return nil, err+ }+ f.features.Fill(ctx, &tempF)+ f.dirCache = tempF.dirCache+ f.root = tempF.root+ return f, fs.ErrorIsFile+ }+ return f, nil+}++// List the objects and directories in dir into entries+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return nil, err+ }++ listing, err := f.listFilesAndFolders(ctx, directoryID)+ if err != nil {+ return nil, err+ }++ for _, folder := range listing.Folders {+ folderName := f.opt.Enc.ToStandardName(folder.Name)+ fullPath := getRemote(dir, folderName)+ modTime := time.Unix(int64(folder.CreatedAt), 0)+ d := fs.NewDir(fullPath, modTime).SetID(folder.ID)+ entries = append(entries, d)+ f.dirCache.Put(fullPath, folder.ID)+ }++ for _, file := range listing.Files {+ fileName := f.opt.Enc.ToStandardName(file.Name)+ remote := getRemote(dir, fileName)+ o := &Object{+ fs: f,+ remote: remote,+ file: file,+ }+ entries = append(entries, o)+ }++ return entries, nil+}++// NewObject finds the Object at remote+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {+ leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)+ if err != nil {+ if err == fs.ErrorDirNotFound {+ return nil, fs.ErrorObjectNotFound+ }+ return nil, err+ }++ listing, err := f.listFilesAndFolders(ctx, directoryID)+ if err != nil {+ return nil, err+ }++ for _, file := range listing.Files {+ if f.opt.Enc.ToStandardName(file.Name) == leaf {+ return &Object{+ fs: f,+ remote: remote,+ file: file,+ }, nil+ }+ }++ return nil, fs.ErrorObjectNotFound+}++// Put uploads a new file+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ existingObj, err := f.NewObject(ctx, src.Remote())+ switch err {+ case nil:+ return existingObj, existingObj.Update(ctx, in, src, options...)+ case fs.ErrorObjectNotFound:+ return f.put(ctx, in, src, options...)+ default:+ return nil, err+ }+}++// put uploads src as a new file+func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ remote := src.Remote()++ _, directoryID, err := f.dirCache.FindPath(ctx, remote, true)+ if err != nil {+ return nil, err+ }++ modTime := src.ModTime(ctx)+ resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)+ if err != nil {+ return nil, err+ }++ return &Object{+ fs: f,+ remote: remote,+ file: fileFromUpload(resp, modTime),+ }, nil+}++// fileFromUpload builds the FileItem for the object an upload just created.+// The single-shot upload response omits updated_at, so the modification time+// is taken from what was sent (modTime) rather than defaulting to the create+// time - it is the time the server stored via the source-mtime header. The+// SHA-256 comes straight from the response (empty for a multipart upload).+func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {+ updatedAt := resp.File.CreatedAt+ if !modTime.IsZero() {+ updatedAt = float64(modTime.Unix())+ }+ return api.FileItem{+ ID: resp.File.ID,+ Name: resp.File.Name,+ SizeBytes: resp.File.SizeBytes,+ MimeType: resp.File.MimeType,+ Extension: resp.File.Extension,+ Region: resp.File.Region,+ CreatedAt: resp.File.CreatedAt,+ UpdatedAt: updatedAt,+ ContentHash: resp.File.ContentHash,+ }+}++// Mkdir makes the directory (container, bucket)+func (f *Fs) Mkdir(ctx context.Context, dir string) error {+ _, err := f.dirCache.FindDir(ctx, dir, true)+ return err+}++// Rmdir removes the directory (container, bucket) if empty+func (f *Fs) Rmdir(ctx context.Context, dir string) error {+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return err+ }++ // The server checks emptiness itself (?if_empty=1) and answers+ // fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.+ err = f.removeFolder(ctx, directoryID, true)+ if err != nil {+ return err+ }++ f.dirCache.FlushDir(dir)+ return nil+}++// Move src to this remote using server-side move operations+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ srcObj, ok := src.(*Object)+ if !ok {+ fs.Debugf(src, "Can't move - not same remote type")+ return nil, fs.ErrorCantMove+ }++ srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)+ _, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)+ if err != nil {+ return nil, err+ }++ // Find destination directory+ dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)+ if err != nil {+ return nil, err+ }++ needsMove := srcDirID != dstDirID+ needsRename := srcLeaf != dstLeaf++ // Moving and renaming in the same request (rather than a moveFile call+ // followed by a separate renameFile call) matters when both are needed:+ // a two-step move-then-rename can transiently place the file's CURRENT+ // name into a destination folder that already holds an unrelated file+ // with that name, even though the FINAL name is free - the server only+ // allows one active name per folder, so that intermediate state 409s.+ if needsMove || needsRename {+ newName := ""+ if needsRename {+ newName = f.opt.Enc.FromStandardName(dstLeaf)+ }+ err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)+ if err != nil {+ return nil, err+ }+ }++ // Return new object. file.Name must reflect the destination leaf, not+ // the source's - a caller that reuses this object as the src of a+ // SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)+ // recomputes srcLeaf from file.Name; a stale name there makes that+ // second move see srcLeaf == dstLeaf and skip the rename entirely,+ // silently leaving the object under the wrong name.+ newFile := srcObj.file+ newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)+ return &Object{+ fs: f,+ remote: remote,+ file: newFile,+ }, nil+}++// DirMove moves src, srcRemote to this remote at dstRemote+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {+ srcFs, ok := src.(*Fs)+ if !ok {+ fs.Debugf(srcFs, "Can't move directory - not same remote type")+ return fs.ErrorCantDirMove+ }++ srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)+ if err != nil {+ return err+ }++ srcLeaf := srcRemote+ if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {+ srcLeaf = srcRemote[idx+1:]+ }+ needsRename := srcLeaf != dstLeaf+ newName := ""+ if needsRename {+ newName = f.opt.Enc.FromStandardName(dstLeaf)+ }++ // Move folder to new parent only if parent changed. The new name goes+ // in the same request, as for files in Move: a move followed by a+ // separate rename can collide with an unrelated folder sharing the+ // CURRENT name in the destination even though the final name is free.+ if srcDirectoryID != dstDirectoryID {+ renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)+ if err != nil {+ return fmt.Errorf("couldn't move directory: %w", err)+ }+ if renamed {+ needsRename = false+ }+ }++ if needsRename {+ err = f.renameFolder(ctx, srcID, newName)+ if err != nil {+ return fmt.Errorf("couldn't rename directory: %w", err)+ }+ }++ srcFs.dirCache.FlushDir(srcRemote)+ return nil+}++// Copy src to this remote using server-side copy operations+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ srcObj, ok := src.(*Object)+ if !ok {+ fs.Debugf(src, "Can't copy - not same remote type")+ return nil, fs.ErrorCantCopy+ }++ dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)+ if err != nil {+ return nil, err+ }++ // The copy API creates a new row named after the source file when+ // copying across folders, which collides with one already at the+ // destination. Copy under a temporary name, then delete the old+ // destination, then move the copy onto the name - so a copy that fails+ // leaves the destination as it was. Deleting it up front lost it+ // (prior art: backend/box).+ existingObj, err := f.NewObject(ctx, remote)+ switch err {+ case nil:+ tempRemote := remote + "-rclone-copy-" + random.String(8)+ fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)+ tempObj, err := f.Copy(ctx, src, tempRemote)+ if err != nil {+ return nil, err+ }+ fs.Debugf(tempRemote, "moving to real name %v", remote)+ if err := existingObj.Remove(ctx); err != nil {+ return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)+ }+ return f.Move(ctx, tempObj, remote)+ case fs.ErrorObjectNotFound:+ // nothing at the destination, so copy straight onto the name+ default:+ return nil, err+ }++ dstName := f.opt.Enc.FromStandardName(dstLeaf)++ resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)+ if err != nil {+ return nil, fmt.Errorf("couldn't copy file: %w", err)+ }++ // Servers that predate the name field ignore it and name the copy after+ // the source, or "Copy of <original>" in the same folder. Rename to the+ // requested destination name if it differs.+ if resp.Name != dstName {+ err = f.renameFile(ctx, resp.FileID, dstName)+ if err != nil {+ return nil, fmt.Errorf("couldn't rename copied file: %w", err)+ }+ resp.Name = dstName+ }++ // A server-side copy is byte-identical to the source and the API+ // preserves the source's modification time and content hash onto the+ // copy, so the returned object must report the SOURCE's modtime and hash,+ // not the copy's wall-clock time. Reading "now" here made the object+ // rclone holds in memory disagree with the one read back from the remote+ // (its Fingerprint check compares the two exactly).+ return &Object{+ fs: f,+ remote: remote,+ file: api.FileItem{+ ID: resp.FileID,+ Name: resp.Name,+ SizeBytes: srcObj.file.SizeBytes,+ MimeType: srcObj.file.MimeType,+ Extension: srcObj.file.Extension,+ Region: srcObj.file.Region,+ CreatedAt: srcObj.file.CreatedAt,+ UpdatedAt: srcObj.file.UpdatedAt,+ ContentHash: srcObj.file.ContentHash,+ },+ }, nil+}++// Purge deletes all the files and the container+func (f *Fs) Purge(ctx context.Context, dir string) error {+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)+ if err != nil {+ return err+ }++ // DELETE /api/folders/:id?permanent=1 recursively deletes all contents+ err = f.removeFolder(ctx, directoryID, false)+ if err != nil {+ return err+ }++ f.dirCache.FlushDir(dir)+ return nil+}++// PublicLink generates a public link to the remote path+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {+ o, err := f.NewObject(ctx, remote)+ if err != nil {+ return "", err+ }+ obj := o.(*Object)++ resp, err := f.createShareLink(ctx, obj.file.ID, expire)+ if err != nil {+ return "", err+ }++ return resp.Link.URL, nil+}++// DirCacheFlush resets the directory cache - used in testing as an+// optional interface+func (f *Fs) DirCacheFlush() {+ f.dirCache.ResetRoot()+}++// About gets quota information from the Fs+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {+ info, err := f.getWorkspaceInfo(ctx)+ if err != nil {+ return nil, err+ }+ usage := &fs.Usage{+ Used: fs.NewUsageValue(info.Storage.Used),+ Total: fs.NewUsageValue(info.Storage.Total),+ Free: fs.NewUsageValue(info.Storage.Free),+ }+ return usage, nil+}++// ------------------------------------------------------------+// Object methods+// ------------------------------------------------------------++// String returns a description of the Object+func (o *Object) String() string {+ return o.remote+}++// Remote returns the remote path+func (o *Object) Remote() string {+ return o.remote+}++// ModTime returns the modification date of the file+func (o *Object) ModTime(ctx context.Context) time.Time {+ if o.file.UpdatedAt > 0 {+ return time.Unix(int64(o.file.UpdatedAt), 0)+ }+ return time.Unix(int64(o.file.CreatedAt), 0)+}++// Size returns the size of the file+func (o *Object) Size() int64 {+ return o.file.SizeBytes+}++// Fs returns read only access to the Fs that this object is part of+func (o *Object) Fs() fs.Info {+ return o.fs+}++// Hash returns the selected checksum of the file+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {+ if t != hash.SHA256 {+ return "", hash.ErrUnsupported+ }+ // May be "" - a multipart-uploaded object carries no server-side hash,+ // which rclone treats as "hash unavailable for this object" (as it does+ // for S3 multipart objects).+ return o.file.ContentHash, nil+}++// Storable says whether this object can be stored+func (o *Object) Storable() bool {+ return true+}++// SetModTime sets the metadata on the object to set the modification date+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {+ return fs.ErrorCantSetModTime+}++// Open opens the file for read+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {+ fs.FixRangeOption(options, o.file.SizeBytes)+ return o.fs.downloadFile(ctx, o.file.ID, options)+}++// Update replaces the file content+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {+ // Find the directory ID for this file+ _, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)+ if err != nil {+ return err+ }++ // Upload as new version of existing file+ fileID := o.file.ID+ modTime := src.ModTime(ctx)+ resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)+ if err != nil {+ return err+ }++ // Update object metadata+ o.file = fileFromUpload(resp, modTime)++ return nil+}++// Remove removes this object+func (o *Object) Remove(ctx context.Context) error {+ return o.fs.deleteFile(ctx, o.file.ID)+}++// MimeType of an Object if known, "" otherwise+func (o *Object) MimeType(ctx context.Context) string {+ return o.file.MimeType+}++// ID returns the ID of the Object if known, or "" if not+func (o *Object) ID() string {+ return o.file.ID+}++func getRemote(dir, fileName string) string {+ if dir == "" {+ return fileName+ }+ return dir + "/" + fileName+}++// Check the interfaces are satisfied+var (+ _ fs.Fs = (*Fs)(nil)+ _ fs.Mover = (*Fs)(nil)+ _ fs.DirMover = (*Fs)(nil)+ _ fs.Copier = (*Fs)(nil)+ _ fs.Purger = (*Fs)(nil)+ _ fs.PublicLinker = (*Fs)(nil)+ _ fs.Abouter = (*Fs)(nil)+ _ fs.DirCacheFlusher = (*Fs)(nil)+ _ dircache.DirCacher = (*Fs)(nil)+ _ fs.Object = (*Object)(nil)+ _ fs.MimeTyper = (*Object)(nil)+ _ fs.IDer = (*Object)(nil)+)diff --git a/backend/dosya/dosya_api.go b/backend/dosya/dosya_api.gonew file mode 100644index 000000000..33f93a65e--- /dev/null+++ b/backend/dosya/dosya_api.go@@ -0,0 +1,880 @@+package dosya++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "path"+ "strconv"+ "strings"+ "time"++ "github.com/rclone/rclone/backend/dosya/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/readers"+ "github.com/rclone/rclone/lib/rest"+ "golang.org/x/sync/errgroup"+)++// sourceMTimeHeader is the request header the upload endpoints read to set a+// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.+const sourceMTimeHeader = "x-dosya-source-mtime"++// modTimeHeaders returns the ExtraHeaders that carry a source modification time+// on an upload, or nil when there is nothing plausible to send. The server+// clamps implausible values itself, but a zero time is never worth a header.+func modTimeHeaders(modTime time.Time) map[string]string {+ if modTime.IsZero() {+ return nil+ }+ return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}+}++// workspaceHintHeader names the workspace a by-id request is about. The API+// maps an id to its workspace through an index the workspace fills about a+// second after a write, so an id this client was handed moments ago can still+// miss; the hint lets the server look for it in the named workspace instead.+// It only locates: every route still checks membership against the workspace+// it resolved, and a workspace-pinned key's hint is ignored unless it matches.+const workspaceHintHeader = "X-Dosya-Workspace"++// byIDHeaders returns the ExtraHeaders every /api/files/:id and+// /api/folders/:id call carries, or nil when no workspace is configured.+func (f *Fs) byIDHeaders() map[string]string {+ if f.opt.WorkspaceID == "" {+ return nil+ }+ return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}+}++// retryErrorCodes is a slice of error codes that we will retry+var retryErrorCodes = []int{+ 429, // Too Many Requests+ 500, // Internal Server Error+ 502, // Bad Gateway+ 503, // Service Unavailable+ 504, // Gateway Timeout+}++// shouldRetry returns a boolean as to whether this resp and err+// deserve to be retried+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if fserrors.ContextError(ctx, &err) {+ return false, err+ }+ retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)+ if retry {+ if d, ok := retryAfter(resp); ok {+ // The API answers 503 + Retry-After while a workspace is being+ // moved between storage homes, and Cloudflare sends it with 429.+ // The pacer's own backoff tops out at maxSleep, far short of that.+ fs.Debugf(nil, "Server asked to retry after %v", d)+ err = pacer.RetryAfterError(err, d)+ }+ }+ return retry, err+}++// maxRetryAfter caps how long a single Retry-After is honoured for+const maxRetryAfter = 5 * time.Minute++// retryAfter reads a 429 or 503 response's Retry-After header, which the API+// sends as whole seconds+func retryAfter(resp *http.Response) (time.Duration, bool) {+ if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {+ return 0, false+ }+ seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))+ if err != nil || seconds <= 0 {+ return 0, false+ }+ return min(time.Duration(seconds)*time.Second, maxRetryAfter), true+}++// errorHandler decodes a non-2xx response into an *api.Error so callers can+// act on the machine-readable error_code rather than the message text.+func errorHandler(resp *http.Response) error {+ body, err := rest.ReadBody(resp)+ if err != nil {+ return fmt.Errorf("error reading error out of body: %w", err)+ }+ apiErr := &api.Error{+ StatusCode: resp.StatusCode,+ Status: resp.Status,+ }+ if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {+ // Not the API's JSON shape (e.g. an edge response that never+ // reached the API) - keep the body text so it isn't lost.+ apiErr.ErrorCode = ""+ apiErr.Message = strings.TrimSpace(string(body))+ }+ return apiErr+}++// The workspace caps the number of in-flight uploads per user. The API+// refuses an upload/init past that cap with a 400 carrying+// error_code "concurrent_upload_limit", and documents it as an error to+// wait on rather than give up on: slots free as uploads finish, and the+// server marks sessions idle for 10 minutes as failed, so waiting up to+// that long guarantees progress unless the workspace really is saturated.+// A 400 is deliberately outside the pacer's retry codes, so the wait+// lives in initUpload instead of shouldRetry.+var (+ concurrentUploadRetryDelay = 5 * time.Second+ concurrentUploadWaitMax = 10 * time.Minute+)++// abortUploadTimeout bounds the abort of a session the client is giving up on,+// which runs outside the transfer's context so a cancelled transfer still+// sends it+const abortUploadTimeout = 10 * time.Second++// isConcurrentUploadLimit reports whether err is the API refusing an+// upload because the workspace's concurrent upload limit is reached+func isConcurrentUploadLimit(err error) bool {+ var apiErr *api.Error+ return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"+}++// listPageSize is the largest page GET /api/files will serve+const listPageSize = 500++// listFilesAndFolders lists files and folders in a directory+//+// The API pages files (at most listPageSize per page) but returns every+// folder on each page, so the files from all pages are gathered and the+// folders are taken from the first page only. Pages are ordered by name,+// which the API tie-breaks on id, so offsets are stable between requests.+func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {+ var listing *api.ListResponse+ for page := 1; ; page++ {+ opts := rest.Opts{+ Method: "GET",+ Path: "/api/files",+ Parameters: map[string][]string{+ "workspace_id": {f.opt.WorkspaceID},+ "per_page": {strconv.Itoa(listPageSize)},+ "page": {strconv.Itoa(page)},+ "sort": {"name"},+ "dir": {"asc"},+ },+ }+ if folderID != "" && folderID != rootID {+ opts.Parameters["folder_id"] = []string{folderID}+ }++ var result api.ListResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't list files: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ if listing == nil {+ listing = &result+ } else {+ listing.Files = append(listing.Files, result.Files...)+ }+ if page >= result.Pagination.TotalPages || len(result.Files) == 0 {+ return listing, nil+ }+ }+}++// createFolder creates a folder+func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {+ request := api.CreateFolderRequest{+ WorkspaceID: f.opt.WorkspaceID,+ Name: name,+ }+ if parentID != "" && parentID != rootID {+ request.ParentID = &parentID+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/api/folders",+ }++ var result api.CreateFolderResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't create folder: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}++// removeFolder deletes a folder+//+// ?permanent=1 trashes and purges a live folder in one request on today's+// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with+// HTTP 409 "Folder is not empty" instead of purging a folder that still+// holds a live file or a live child folder; that 409 is turned into+// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API+// error, and it is never retried (409 is not one of retryErrorCodes). A big+// subtree is purged in bounded passes (202, complete=false), so keep+// calling until the purge is complete. An older server that ignores+// permanent=1 only trashes on the first call (answering permanent=false),+// so the loop's second pass sends the plain DELETE that purges it.+func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {+ parameters := map[string][]string{"permanent": {"1"}}+ if ifEmpty {+ parameters["if_empty"] = []string{"1"}+ }+ opts := rest.Opts{+ Method: "DELETE",+ Path: "/api/folders/" + folderID,+ Parameters: parameters,+ ExtraHeaders: f.byIDHeaders(),+ }++ for pass := 0; ; pass++ {+ var result api.DeleteFolderResponse+ var statusCode int+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ if resp != nil {+ statusCode = resp.StatusCode+ }+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ if ifEmpty && statusCode == http.StatusConflict {+ return fs.ErrorDirectoryNotEmpty+ }+ if pass == 0 {+ return fmt.Errorf("couldn't remove folder: %w", err)+ }+ return fmt.Errorf("couldn't permanently remove folder: %w", err)+ }+ if !result.OK {+ return fmt.Errorf("API error: %s", result.Error)+ }+ if result.Permanent && result.Complete {+ return nil+ }+ if pass >= maxFolderPurgePasses {+ return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)+ }+ }+}++// maxFolderPurgePasses bounds the DELETE calls spent purging one folder+const maxFolderPurgePasses = 1000++// renameFolder renames a folder+func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {+ request := api.RenameFolderRequest{Name: newName}+ opts := rest.Opts{+ Method: "PUT",+ Path: "/api/folders/" + folderID + "/rename",+ ExtraHeaders: f.byIDHeaders(),+ }++ var result api.Response+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return fmt.Errorf("couldn't rename folder: %w", err)+ }+ if !result.OK {+ return fmt.Errorf("API error: %s", result.Error)+ }+ return nil+}++// moveFolder moves a folder to a new parent+func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {+ request := api.MoveFolderRequest{Name: newName}+ if newParentID != "" && newParentID != rootID {+ request.ParentID = &newParentID+ }+ opts := rest.Opts{+ Method: "PUT",+ Path: "/api/folders/" + folderID + "/move",+ ExtraHeaders: f.byIDHeaders(),+ }++ var result api.MoveFolderResponse+ err = f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return false, fmt.Errorf("couldn't move folder: %w", err)+ }+ if !result.OK {+ return false, fmt.Errorf("API error: %s", result.Error)+ }+ // Servers that predate the name field ignore it and answer without a+ // name, leaving the caller to rename separately+ return newName != "" && result.Name == newName, nil+}++// deleteFile deletes a file+//+// ?permanent=1 trashes and purges it in one request on today's API. An+// older server that ignores the flag only soft-deletes it, answering+// permanent=false, so a second DELETE on the now soft-deleted file purges+// it from the database and R2, as it always has.+func (f *Fs) deleteFile(ctx context.Context, fileID string) error {+ opts := rest.Opts{+ Method: "DELETE",+ Path: "/api/files/" + fileID,+ Parameters: map[string][]string{"permanent": {"1"}},+ ExtraHeaders: f.byIDHeaders(),+ }++ var result api.DeleteFileResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return fmt.Errorf("couldn't delete file: %w", err)+ }+ if !result.OK {+ return fmt.Errorf("API error: %s", result.Error)+ }++ // Answered into its own value so a field the second response omits+ // can't be read off the first.+ if !result.Permanent {+ var permanent api.DeleteFileResponse+ err = f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return fmt.Errorf("couldn't permanently delete file: %w", err)+ }+ if !permanent.OK {+ return fmt.Errorf("API error: %s", permanent.Error)+ }+ }+ return nil+}++// renameFile renames a file+func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {+ request := api.RenameFileRequest{Name: newName}+ opts := rest.Opts{+ Method: "PUT",+ Path: "/api/files/" + fileID + "/rename",+ ExtraHeaders: f.byIDHeaders(),+ }++ var result api.RenameFileResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return fmt.Errorf("couldn't rename file: %w", err)+ }+ if !result.OK {+ return fmt.Errorf("API error: %s", result.Error)+ }+ return nil+}++// moveFile moves a file to a new folder and, if newName is non-empty,+// renames it in the same request+func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {+ request := api.MoveFileRequest{}+ if folderID != "" && folderID != rootID {+ request.FolderID = &folderID+ }+ if newName != "" {+ request.Name = &newName+ }+ opts := rest.Opts{+ Method: "PUT",+ Path: "/api/files/" + fileID + "/move",+ ExtraHeaders: f.byIDHeaders(),+ }++ var result api.Response+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return fmt.Errorf("couldn't move file: %w", err)+ }+ if !result.OK {+ return fmt.Errorf("API error: %s", result.Error)+ }+ return nil+}++// copyFile copies a file to a folder+func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {+ request := api.CopyFileRequest{Name: name}+ if folderID != "" && folderID != rootID {+ request.FolderID = &folderID+ }+ opts := rest.Opts{+ Method: "POST",+ Path: "/api/files/" + fileID + "/copy",+ ExtraHeaders: f.byIDHeaders(),+ }++ var result api.CopyFileResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't copy file: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}++// downloadFile downloads a file by its ID+//+// The download endpoint answers 302 with a presigned R2 URL in Location, so+// this is two calls. Both go through the rest client, which is built on+// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,+// --user-agent, --bind, --dump and the proxy settings apply to downloads as+// they do to every other call, and errorHandler and shouldRetry decide the+// errors and the retries uniformly.+func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {+ // NoRedirect stops the client following the 302 with the API's+ // Authorization header attached. A 302 is not a 2xx, so Call reports it+ // as an error (having read and closed the body); the Location header+ // is what actually matters, and finding it clears the error, as in+ // backend/linkbox.+ opts := rest.Opts{+ Method: "GET",+ Path: "/api/files/" + fileID + "/download",+ ExtraHeaders: f.byIDHeaders(),+ NoRedirect: true,+ }+ var downloadURL string+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.Call(ctx, &opts)+ if resp != nil {+ if location := resp.Header.Get("Location"); location != "" {+ downloadURL, err = location, nil+ }+ }+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't get download URL: %w", err)+ }+ if downloadURL == "" {+ return nil, errors.New("no download URL in redirect response")+ }++ // Fetch the object from the presigned URL. It carries its own+ // credentials and R2 refuses a request that also has an Authorization+ // header, so the client-level bearer token is removed by setting the+ // header empty: the rest client skips a header with an empty value.+ opts = rest.Opts{+ Method: "GET",+ RootURL: downloadURL,+ Options: options,+ ExtraHeaders: map[string]string{"Authorization": ""},+ }+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ var err error+ resp, err = f.rest.Call(ctx, &opts)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't download file: %w", err)+ }+ return resp.Body, nil+}++// uploadSmallFile uploads a file smaller than 10MB in a single request+func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {+ opts := rest.Opts{+ Method: "PUT",+ Path: "/api/upload/" + sessionID,+ Body: in,+ ContentLength: &size,+ ContentType: "application/octet-stream",+ ExtraHeaders: modTimeHeaders(modTime),+ }++ // The body is a stream which can't be rewound, so a low level retry+ // would resend whatever is left of it - nothing - and the server+ // would store that as an empty file. Don't retry here; rclone's high+ // level retries redo the whole transfer instead.+ var result api.UploadCompleteResponse+ err := f.pacer.CallNoRetry(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't upload file: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}++// uploadPart uploads a single part of a multipart upload+func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {+ size := int64(len(part))+ opts := rest.Opts{+ Method: "PUT",+ Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),+ ContentLength: &size,+ ContentType: "application/octet-stream",+ }++ var result api.UploadPartResponse+ err := f.pacer.Call(func() (bool, error) {+ // A fresh reader per attempt so a retry resends the whole part+ opts.Body = bytes.NewReader(part)+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}++// completeUpload finalizes a multipart upload+func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {+ opts := rest.Opts{+ Method: "POST",+ Path: "/api/upload/" + sessionID + "/complete",+ ExtraHeaders: modTimeHeaders(modTime),+ }++ var result api.UploadCompleteResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't complete upload: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}++// initUpload initializes a new upload session+func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {+ request := api.UploadInitRequest{+ WorkspaceID: f.opt.WorkspaceID,+ FileName: fileName,+ FileSize: fileSize,+ MimeType: mimeType,+ FileID: fileID,+ }+ if folderID != "" && folderID != rootID {+ request.FolderID = &folderID+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/api/upload/init",+ }++ var result api.UploadInitResponse+ deadline := time.Now().Add(concurrentUploadWaitMax)+ for {+ result = api.UploadInitResponse{}+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err == nil {+ break+ }+ if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {+ return nil, fmt.Errorf("couldn't init upload: %w", err)+ }+ fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)+ select {+ case <-ctx.Done():+ return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())+ case <-time.After(concurrentUploadRetryDelay):+ }+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}++// uploadParts reads the source part by part and uploads the parts.+//+// Part 1 goes on its own: the server creates the R2 multipart upload (and a+// new file's id and key) when it receives the first part, so parts sent+// before that has finished would each start an upload of their own. The+// rest go out opt.UploadConcurrency at a time. A part is only read from the+// source once a buffer is free, so at most that many parts are held in+// memory, and a short source is still found by the reader before the part+// it could not fill is sent.+func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {+ readPart := func(buf []byte, partNum int) ([]byte, error) {+ currentPartSize := partSize+ if partNum == totalParts {+ currentPartSize = size - partSize*int64(totalParts-1)+ }+ part := buf[:currentPartSize]+ if _, err := io.ReadFull(in, part); err != nil {+ if err == io.EOF || err == io.ErrUnexpectedEOF {+ return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)+ }+ return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)+ }+ return part, nil+ }++ concurrency := max(f.opt.UploadConcurrency, 1)+ buffers := make(chan []byte, concurrency)+ buffers <- make([]byte, partSize)+ allocated := 1++ first := <-buffers+ part, err := readPart(first, 1)+ if err != nil {+ return err+ }+ if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {+ return err+ }+ buffers <- first++ // A failed read has to take the parts already in flight down with it,+ // or they upload to completion while the error waits on Wait.+ ctx, cancel := context.WithCancel(ctx)+ defer cancel()+ g, gCtx := errgroup.WithContext(ctx)+ var readErr error+ for partNum := 2; partNum <= totalParts; partNum++ {+ // Take a free buffer, allocating one while under the limit.+ var buf []byte+ select {+ case buf = <-buffers:+ default:+ if allocated < concurrency {+ buf = make([]byte, partSize)+ allocated+++ } else {+ select {+ case buf = <-buffers:+ case <-gCtx.Done():+ }+ }+ }+ if buf == nil {+ break // a part failed; its error comes back from Wait+ }+ part, err := readPart(buf, partNum)+ if err != nil {+ readErr = err+ cancel()+ break+ }+ g.Go(func() error {+ defer func() { buffers <- buf }()+ _, err := f.uploadPart(gCtx, part, sessionID, partNum)+ return err+ })+ }+ err = g.Wait()+ if readErr != nil {+ return readErr+ }+ return err+}++// abortUpload retires an upload session the client is giving up on.+//+// Best effort: the server reclaims an abandoned session itself, so a failure+// here - including an older server with no such route - costs the slot the+// wait it would have cost anyway and is not worth failing an upload over.+// Sent on a context of its own so a cancelled transfer still gives its slot+// back rather than leaving it to the sweep.+func (f *Fs) abortUpload(ctx context.Context, sessionID string) {+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)+ defer cancel()+ opts := rest.Opts{+ Method: "DELETE",+ Path: "/api/upload/" + sessionID,+ NoResponse: true,+ }+ err := f.pacer.CallNoRetry(func() (bool, error) {+ resp, err := f.rest.Call(ctx, &opts)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)+ }+}++// uploadFile handles the full upload flow (small or multipart)+func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {+ // Both refusals live here so a new file and a new version of an existing+ // one answer the same way: upload/init needs a byte count up front, and+ // the server stores no zero-length object.+ if size < 0 {+ return nil, errors.New("can't upload files with unknown size")+ }+ if size == 0 {+ return nil, fs.ErrorCantUploadEmptyFiles+ }++ leaf := f.opt.Enc.FromStandardName(path.Base(remote))++ initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)+ if err != nil {+ return nil, err+ }+ // A session abandoned here holds one of the workspace's upload slots+ // until the server's sweep retires it ten minutes later, and rclone's+ // high-level retry opens a fresh session rather than resuming this one -+ // so without this, a few failed attempts use up every slot.+ defer func() {+ if err != nil {+ f.abortUpload(ctx, initResp.SessionID)+ }+ }()++ // Count what the source actually supplies, so a source which ends+ // early is reported as an error rather than stored as a truncated+ // file.+ counter := readers.NewCountingReader(in)++ // Small file upload (single PUT, no resumable)+ if initResp.Resumable == nil {+ resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)+ if err != nil {+ return nil, err+ }+ if got := int64(counter.BytesRead()); got != size {+ return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)+ }+ return resp, nil+ }++ // Multipart upload. Each part is read fully into memory before it is+ // sent, so a short source is caught before the part goes out and a+ // part which has to be retried is resent with all of its bytes.+ partSize := initResp.Resumable.PartSize+ totalParts := initResp.Resumable.TotalParts+ // Both bounds matter. partSize*(totalParts-1) >= size means a part would+ // have nothing to carry; size > partSize*totalParts means the last part+ // is bigger than partSize, and readPart would slice past its buffer and+ // panic.+ if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {+ return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)+ }+ if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {+ return nil, err+ }++ return f.completeUpload(ctx, initResp.SessionID, modTime)+}++// createShareLink creates a public share link for a file+func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {+ request := api.ShareLinkRequest{}+ if expire < fs.DurationOff {+ days := int(time.Duration(expire).Hours() / 24)+ if days < 1 {+ days = 1+ }+ request.ExpiresInDays = &days+ }++ opts := rest.Opts{+ Method: "POST",+ Path: "/api/files/" + fileID + "/share",+ ExtraHeaders: f.byIDHeaders(),+ }++ var result api.ShareLinkResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't create share link: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}++// getWorkspaceInfo fetches workspace storage info+// listWorkspaces lists the workspaces this credential can reach. An API key+// pinned to a workspace sees exactly one: its own.+func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "/api/workspaces",+ }++ var result api.WorkspaceListResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't list workspaces: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return result.Workspaces, nil+}++func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {+ opts := rest.Opts{+ Method: "GET",+ Path: "/api/workspaces/" + f.opt.WorkspaceID,+ }++ var result api.WorkspaceInfoResponse+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("couldn't get workspace info: %w", err)+ }+ if !result.OK {+ return nil, fmt.Errorf("API error: %s", result.Error)+ }+ return &result, nil+}diff --git a/backend/dosya/dosya_compat_test.go b/backend/dosya/dosya_compat_test.gonew file mode 100644index 000000000..ced911be7--- /dev/null+++ b/backend/dosya/dosya_compat_test.go@@ -0,0 +1,235 @@+package dosya++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "testing"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/dircache"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestShouldRetryHonoursRetryAfter(t *testing.T) {+ ctx := context.Background()+ for _, tc := range []struct {+ name string+ status int+ header string+ want time.Duration+ ok bool+ }{+ {"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},+ {"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},+ {"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},+ {"missing header", http.StatusServiceUnavailable, "", 0, false},+ {"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},+ {"502 ignores the header", http.StatusBadGateway, "30", 0, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)+ if tc.header != "" {+ resp.Header.Set("Retry-After", tc.header)+ }+ retry, err := shouldRetry(ctx, resp, errorHandler(resp))+ assert.True(t, retry)+ d, ok := pacer.IsRetryAfter(err)+ assert.Equal(t, tc.ok, ok)+ if tc.ok {+ assert.Equal(t, tc.want, d)+ }+ })+ }+}++func TestShouldRetryDoesNotRetryConflict(t *testing.T) {+ resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)+ resp.Header.Set("Retry-After", "30")+ retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))+ assert.False(t, retry)+ _, ok := pacer.IsRetryAfter(err)+ assert.False(t, ok)+}++func TestRemoveFolderPurgesPermanently(t *testing.T) {+ // a big subtree purges in bounded passes: 202 complete=false repeats+ // the same request until complete=true+ answers := []struct {+ status int+ body string+ }{+ {http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},+ {http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},+ }+ calls := 0+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ require.Equal(t, "DELETE", r.Method)+ require.Equal(t, "/api/folders/fld_1", r.URL.Path)+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))+ assert.Equal(t, "", r.URL.Query().Get("if_empty"))+ require.Less(t, calls, len(answers), "DELETE sent after the purge completed")+ a := answers[calls]+ calls+++ return jsonResp(a.status, a.body), nil+ }))++ require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))+ assert.Equal(t, len(answers), calls)+}++func TestRemoveFolderStopsOnPurgeError(t *testing.T) {+ calls := 0+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ calls+++ return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil+ }))++ err := f.removeFolder(context.Background(), "fld_1", false)+ require.Error(t, err)+ assert.Contains(t, err.Error(), "locked file")+ assert.Equal(t, 1, calls)+}++func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {+ calls := 0+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ calls+++ assert.Equal(t, "/api/folders/fld_1", r.URL.Path)+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))+ assert.Equal(t, "1", r.URL.Query().Get("if_empty"))+ return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil+ }))++ err := f.removeFolder(context.Background(), "fld_1", true)+ assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)+ assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")+}++func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {+ calls := 0+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ calls+++ assert.Equal(t, "1", r.URL.Query().Get("permanent"))+ assert.Equal(t, "1", r.URL.Query().Get("if_empty"))+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil+ }))++ require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))+ assert.Equal(t, 1, calls)+}++func TestRmdirMakesOneRequest(t *testing.T) {+ calls := 0+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ calls+++ assert.Equal(t, "DELETE", r.Method)+ assert.Equal(t, "/api/folders/fld_1", r.URL.Path)+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))+ assert.Equal(t, "1", r.URL.Query().Get("if_empty"))+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil+ }))+ f.dirCache = dircache.New("", rootID, f)+ require.NoError(t, f.dirCache.FindRoot(context.Background(), false))+ f.dirCache.Put("sub", "fld_1")++ require.NoError(t, f.Rmdir(context.Background(), "sub"))+ assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")+}++func TestRmdirMapsNonEmptyConflict(t *testing.T) {+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil+ }))+ f.dirCache = dircache.New("", rootID, f)+ require.NoError(t, f.dirCache.FindRoot(context.Background(), false))+ f.dirCache.Put("sub", "fld_1")++ err := f.Rmdir(context.Background(), "sub")+ assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)+}++func TestDeleteFileSendsPermanentFlag(t *testing.T) {+ for _, tc := range []struct {+ name string+ permanent bool+ wantCalls int+ }{+ {"today's API purges in one call", true, 1},+ {"an older server ignores the flag and still gets a second DELETE", false, 2},+ } {+ t.Run(tc.name, func(t *testing.T) {+ calls := 0+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ calls+++ assert.Equal(t, "DELETE", r.Method)+ assert.Equal(t, "/api/files/fil_1", r.URL.Path)+ assert.Equal(t, "1", r.URL.Query().Get("permanent"))+ return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil+ }))++ require.NoError(t, f.deleteFile(context.Background(), "fil_1"))+ assert.Equal(t, tc.wantCalls, calls)+ })+ }+}++// decodeBody reads a JSON request body into a map+func decodeBody(t *testing.T, r *http.Request) map[string]any {+ b, err := io.ReadAll(r.Body)+ require.NoError(t, err)+ var m map[string]any+ require.NoError(t, json.Unmarshal(b, &m))+ return m+}++func TestMoveFolderSendsName(t *testing.T) {+ for _, tc := range []struct {+ name string+ newName string+ respBody string+ wantRenamed bool+ }{+ {"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},+ {"older server ignores name", "dst", `{"ok":true}`, false},+ {"no rename requested", "", `{"ok":true,"name":"src"}`, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)+ body := decodeBody(t, r)+ assert.Equal(t, "fld_parent", body["parent_id"])+ if tc.newName == "" {+ assert.NotContains(t, body, "name")+ } else {+ assert.Equal(t, tc.newName, body["name"])+ }+ return jsonResp(http.StatusOK, tc.respBody), nil+ }))++ renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)+ require.NoError(t, err)+ assert.Equal(t, tc.wantRenamed, renamed)+ })+ }+}++func TestCopyFileSendsName(t *testing.T) {+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)+ body := decodeBody(t, r)+ assert.Equal(t, "fld_dst", body["folder_id"])+ assert.Equal(t, "final.txt", body["name"])+ return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil+ }))++ resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")+ require.NoError(t, err)+ assert.Equal(t, "fil_2", resp.FileID)+ assert.Equal(t, "final.txt", resp.Name)+}diff --git a/backend/dosya/dosya_hash_modtime_test.go b/backend/dosya/dosya_hash_modtime_test.gonew file mode 100644index 000000000..4b79b7aed--- /dev/null+++ b/backend/dosya/dosya_hash_modtime_test.go@@ -0,0 +1,171 @@+package dosya++import (+ "context"+ "io"+ "net/http"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/rclone/rclone/backend/dosya/api"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/lib/dircache"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {+ f := &Fs{}+ assert.Equal(t, time.Second, f.Precision())+ assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")+ // Exactly SHA-256, nothing else.+ assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())+}++func TestObjectHashReturnsContentHash(t *testing.T) {+ o := &Object{file: api.FileItem{ContentHash: "abc123"}}++ got, err := o.Hash(context.Background(), hash.SHA256)+ require.NoError(t, err)+ assert.Equal(t, "abc123", got)++ // A multipart object with no server-side hash yields "" and no error,+ // exactly as an S3 multipart object does.+ empty := &Object{file: api.FileItem{ContentHash: ""}}+ got, err = empty.Hash(context.Background(), hash.SHA256)+ require.NoError(t, err)+ assert.Equal(t, "", got)++ // Any other algorithm is unsupported.+ _, err = o.Hash(context.Background(), hash.MD5)+ assert.ErrorIs(t, err, hash.ErrUnsupported)+}++func TestObjectModTimeUsesUpdatedAt(t *testing.T) {+ mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)+ o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}+ assert.True(t, mt.Equal(o.ModTime(context.Background())))+}++func TestModTimeHeaders(t *testing.T) {+ assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")++ mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)+ h := modTimeHeaders(mt)+ require.NotNil(t, h)+ assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])+}++func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {+ resp := &api.UploadCompleteResponse{}+ resp.File.ID = "fil_1"+ resp.File.Name = "x.txt"+ resp.File.SizeBytes = 10+ resp.File.CreatedAt = 1_000+ resp.File.ContentHash = "deadbeef"++ mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)+ item := fileFromUpload(resp, mt)+ assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")+ assert.Equal(t, "deadbeef", item.ContentHash)+ assert.Equal(t, float64(1_000), item.CreatedAt)++ // A zero modtime falls back to the create time.+ item = fileFromUpload(resp, time.Time{})+ assert.Equal(t, float64(1_000), item.UpdatedAt)+}++// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a+// fake transport and asserts the source-mtime header actually rides on the PUT.+func TestUploadSendsSourceMTimeHeader(t *testing.T) {+ var putMTime string+ rt := rtFunc(func(r *http.Request) (*http.Response, error) {+ if r.Body != nil {+ _, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances+ }+ switch {+ case r.Method == "POST" && r.URL.Path == "/api/upload/init":+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil+ case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":+ putMTime = r.Header.Get(sourceMTimeHeader)+ return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil+ }+ t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)+ return nil, nil+ })+ f := newFakeFs(rt)++ mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)+ resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)+ require.NoError(t, err)+ assert.Equal(t, "h", resp.File.ContentHash)+ assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")+}++// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on+// the completion request for a multipart upload (the server stores the modtime+// there, not on the individual parts).+func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {+ var completeMTime string+ rt := rtFunc(func(r *http.Request) (*http.Response, error) {+ if r.Body != nil {+ _, _ = io.ReadAll(r.Body)+ }+ switch {+ case r.URL.Path == "/api/upload/init":+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil+ case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):+ return jsonResp(http.StatusOK, `{"ok":true}`), nil+ case r.URL.Path == "/api/upload/upl_1/complete":+ completeMTime = r.Header.Get(sourceMTimeHeader)+ return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil+ }+ t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)+ return nil, nil+ })+ f := newFakeFs(rt)++ mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)+ _, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)+ require.NoError(t, err)+ assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")+}++// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a+// server-side Copy returns: it must report the SOURCE's modtime and hash (the+// copy is byte-identical and the API preserves both), not the copy's wall+// clock. Reading "now" here made rclone's in-memory object disagree with the+// one read back from the remote and failed FsCopy's fingerprint check.+func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {+ rt := rtFunc(func(r *http.Request) (*http.Response, error) {+ switch {+ case r.Method == "GET" && r.URL.Path == "/api/files":+ // NewObject's existence probe on the destination: nothing there.+ return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil+ case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):+ return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil+ }+ t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)+ return nil, nil+ })+ f := newFakeFs(rt)+ f.dirCache = dircache.New("", rootID, f)++ mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)+ ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)+ src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{+ ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",+ CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",+ }}++ dst, err := f.Copy(context.Background(), src, "f-copy.txt")+ require.NoError(t, err)++ assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")+ assert.Equal(t, int64(100), dst.Size())+ got, err := dst.Hash(context.Background(), hash.SHA256)+ require.NoError(t, err)+ assert.Equal(t, "srchash", got, "copy keeps the source hash")+}diff --git a/backend/dosya/dosya_internal_test.go b/backend/dosya/dosya_internal_test.gonew file mode 100644index 000000000..33bd5382d--- /dev/null+++ b/backend/dosya/dosya_internal_test.go@@ -0,0 +1,156 @@+package dosya++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rclone/rclone/backend/dosya/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`++func TestErrorHandlerDecodesAPIError(t *testing.T) {+ resp := &http.Response{+ StatusCode: http.StatusBadRequest,+ Status: "400 Bad Request",+ Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),+ }+ err := errorHandler(resp)+ var apiErr *api.Error+ require.ErrorAs(t, err, &apiErr)+ assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)+ assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)+ assert.Equal(t, 5, apiErr.MaxConcurrentUploads)+ assert.Contains(t, err.Error(), "uploads in progress")+ assert.Contains(t, err.Error(), "concurrent_upload_limit")+}++func TestErrorHandlerNonJSONBody(t *testing.T) {+ // e.g. an edge (Cloudflare) response that never reached the API+ resp := &http.Response{+ StatusCode: http.StatusTooManyRequests,+ Status: "429 Too Many Requests",+ Body: io.NopCloser(strings.NewReader("error code: 1015\n")),+ }+ err := errorHandler(resp)+ var apiErr *api.Error+ require.ErrorAs(t, err, &apiErr)+ assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)+ assert.Equal(t, "", apiErr.ErrorCode)+ assert.Contains(t, err.Error(), "429")+ assert.Contains(t, err.Error(), "error code: 1015")+}++func TestShouldRetryAPIErrors(t *testing.T) {+ ctx := context.Background()++ // The concurrent upload limit is handled by initUpload's slot wait,+ // not by the pacer's low level retries.+ limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}+ retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)+ assert.False(t, retry)+ assert.Equal(t, limitErr, err)++ retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})+ assert.True(t, retry)++ retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})+ assert.False(t, retry)+}++// swapWaitTimings shortens the slot wait timings for a test and returns a+// function which restores them+func swapWaitTimings(delay, max time.Duration) func() {+ oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax+ concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max+ return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }+}++// newTestFs makes an Fs which talks to srv+func newTestFs(srv *httptest.Server) *Fs {+ ctx := context.Background()+ f := &Fs{+ name: "test",+ opt: Options{WorkspaceID: "ws_test"},+ pacer: fs.NewPacer(ctx, pacer.NewDefault(+ pacer.MinSleep(time.Millisecond),+ pacer.MaxSleep(time.Millisecond),+ pacer.DecayConstant(decayConstant),+ pacer.AttackConstant(attackConstant),+ )),+ }+ f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)+ return f+}++// limitServer answers the first n init calls with the concurrent upload+// limit error and the rest with success+func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ assert.Equal(t, "/api/upload/init", r.URL.Path)+ w.Header().Set("Content-Type", "application/json")+ if calls.Add(1) <= n {+ w.WriteHeader(http.StatusBadRequest)+ _, _ = w.Write([]byte(concurrentLimitBody))+ return+ }+ w.WriteHeader(http.StatusCreated)+ _, _ = w.Write([]byte(`{"ok":true}`))+ }))+}++func TestInitUploadWaitsForFreeSlot(t *testing.T) {+ defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()+ var calls atomic.Int32+ srv := limitServer(t, 2, &calls)+ defer srv.Close()+ f := newTestFs(srv)++ resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)+ require.NoError(t, err)+ assert.True(t, resp.OK)+ assert.Equal(t, int32(3), calls.Load())+}++func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {+ defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()+ var calls atomic.Int32+ srv := limitServer(t, 1<<30, &calls)+ defer srv.Close()+ f := newTestFs(srv)++ _, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)+ require.Error(t, err)+ var apiErr *api.Error+ require.ErrorAs(t, err, &apiErr)+ assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)+ assert.Contains(t, err.Error(), "couldn't init upload")+ assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")+}++func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {+ defer swapWaitTimings(time.Minute, time.Hour)()+ var calls atomic.Int32+ srv := limitServer(t, 1<<30, &calls)+ defer srv.Close()+ f := newTestFs(srv)++ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)+ defer cancel()+ start := time.Now()+ _, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)+ require.Error(t, err)+ assert.ErrorIs(t, err, context.DeadlineExceeded)+ assert.Less(t, time.Since(start), 5*time.Second)+}diff --git a/backend/dosya/dosya_list_test.go b/backend/dosya/dosya_list_test.gonew file mode 100644index 000000000..545b7d5d1--- /dev/null+++ b/backend/dosya/dosya_list_test.go@@ -0,0 +1,68 @@+package dosya++import (+ "context"+ "fmt"+ "net/http"+ "strconv"+ "strings"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// fakeListServer serves GET /api/files the way the API does: files paged at+// per_page, every folder repeated on each page+func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {+ return func(r *http.Request) (*http.Response, error) {+ require.Equal(t, "/api/files", r.URL.Path)+ q := r.URL.Query()+ assert.Equal(t, "name", q.Get("sort"))+ assert.Equal(t, "asc", q.Get("dir"))+ page, _ := strconv.Atoi(q.Get("page"))+ perPage, _ := strconv.Atoi(q.Get("per_page"))+ require.Positive(t, page)+ require.Positive(t, perPage)+ *pages = append(*pages, page)++ var files []string+ for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {+ files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))+ }+ totalPages := max(1, (totalFiles+perPage-1)/perPage)+ body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`++ `"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,+ strings.Join(files, ","), page, perPage, totalFiles, totalPages)+ return jsonResp(http.StatusOK, body), nil+ }+}++func TestListFilesAndFoldersFollowsPages(t *testing.T) {+ var pages []int+ f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))++ listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")+ require.NoError(t, err)++ assert.Equal(t, []int{1, 2, 3}, pages)+ require.Len(t, listing.Files, 2*listPageSize+7)+ assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)+ // folders repeat on every page and must not be duplicated+ assert.Len(t, listing.Folders, 1)+}++func TestListFilesAndFoldersSinglePage(t *testing.T) {+ for _, total := range []int{0, 1, listPageSize} {+ t.Run(strconv.Itoa(total), func(t *testing.T) {+ var pages []int+ f := newFakeFs(fakeListServer(t, total, &pages))++ listing, err := f.listFilesAndFolders(context.Background(), rootID)+ require.NoError(t, err)++ assert.Equal(t, []int{1}, pages)+ assert.Len(t, listing.Files, total)+ })+ }+}diff --git a/backend/dosya/dosya_parallel_test.go b/backend/dosya/dosya_parallel_test.gonew file mode 100644index 000000000..05b58436d--- /dev/null+++ b/backend/dosya/dosya_parallel_test.go@@ -0,0 +1,153 @@+package dosya++import (+ "context"+ "fmt"+ "io"+ "net/http"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// parallelUploadServer answers the multipart endpoints without holding a+// lock across a request, so parts really do overlap, and records what the+// ordering rules below are about: how many parts were in flight at once, and+// whether any part started before part 1 had finished.+//+// Part 1 matters because the API creates the R2 multipart upload (and, for a+// new file, its id and key) on the first part it sees. Parts that race ahead+// of part 1 would each create their own upload, and only one of those would+// be completed.+type parallelUploadServer struct {+ t *testing.T+ resumable string+ partDelay time.Duration+ failPart int // answer this part with a 400, 0 for none+ mu sync.Mutex+ inFlight int+ maxFlight int+ part1Done bool+ earlyParts []int // parts that started before part 1 finished+ parts map[int]string+ completes int+ aborts int+}++func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {+ var body []byte+ if r.Body != nil {+ var err error+ body, err = io.ReadAll(r.Body)+ require.NoError(s.t, err)+ }+ path := r.URL.Path+ switch {+ case r.Method == "POST" && path == "/api/upload/init":+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil+ case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):+ var n int+ _, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)+ require.NoError(s.t, err)++ s.mu.Lock()+ if n != 1 && !s.part1Done {+ s.earlyParts = append(s.earlyParts, n)+ }+ s.inFlight+++ s.maxFlight = max(s.maxFlight, s.inFlight)+ s.mu.Unlock()++ time.Sleep(s.partDelay)++ s.mu.Lock()+ s.inFlight--+ if n == 1 {+ s.part1Done = true+ }+ s.parts[n] = string(body)+ fail := n == s.failPart+ s.mu.Unlock()++ if fail {+ return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil+ }+ return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil+ case r.Method == "POST" && path == "/api/upload/upl_1/complete":+ s.mu.Lock()+ s.completes+++ s.mu.Unlock()+ return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil+ case r.Method == "DELETE" && path == "/api/upload/upl_1":+ s.mu.Lock()+ s.aborts+++ s.mu.Unlock()+ return jsonResp(http.StatusOK, `{"ok":true}`), nil+ }+ s.t.Fatalf("unexpected request %s %s", r.Method, path)+ return nil, nil+}++func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {+ return ¶llelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}+}++const tenParts = `{"part_size":2,"total_parts":10}`++func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {+ srv := newParallelServer(t, tenParts)+ f := newFakeFs(rtFunc(srv.roundTrip))+ f.opt.UploadConcurrency = 4+ content := "aabbccddeeffgghhiijj"++ _, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.NoError(t, err)++ assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")+ assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")+ assert.Equal(t, 1, srv.completes)+ for i := 1; i <= 10; i++ {+ assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)+ }+}++func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {+ srv := newParallelServer(t, tenParts)+ f := newFakeFs(rtFunc(srv.roundTrip))+ f.opt.UploadConcurrency = 1++ _, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.NoError(t, err)+ assert.Equal(t, 1, srv.maxFlight)+ assert.Equal(t, 1, srv.completes)+}++func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {+ srv := newParallelServer(t, tenParts)+ srv.failPart = 6+ f := newFakeFs(rtFunc(srv.roundTrip))+ f.opt.UploadConcurrency = 4++ _, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.Error(t, err)+ assert.Contains(t, err.Error(), "part 6")+ assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")+}++func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {+ // 20 bytes declared, 15 supplied: the short read is found by the+ // reader, whichever parts are already in flight, and nothing completes.+ srv := newParallelServer(t, tenParts)+ f := newFakeFs(rtFunc(srv.roundTrip))+ f.opt.UploadConcurrency = 4++ _, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.Error(t, err)+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)+ assert.Equal(t, 0, srv.completes)+ assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")+}diff --git a/backend/dosya/dosya_test.go b/backend/dosya/dosya_test.gonew file mode 100644index 000000000..24a92cb8d--- /dev/null+++ b/backend/dosya/dosya_test.go@@ -0,0 +1,15 @@+// Test Dosya filesystem interface+package dosya++import (+ "testing"++ "github.com/rclone/rclone/fstest/fstests"+)++// TestIntegration runs integration tests against the remote+func TestIntegration(t *testing.T) {+ fstests.Run(t, &fstests.Opt{+ RemoteName: "TestDosya:",+ })+}diff --git a/backend/dosya/dosya_upload_test.go b/backend/dosya/dosya_upload_test.gonew file mode 100644index 000000000..2b43ee3d6--- /dev/null+++ b/backend/dosya/dosya_upload_test.go@@ -0,0 +1,247 @@+package dosya++import (+ "context"+ "fmt"+ "io"+ "net/http"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// rtFunc adapts a function into an http.RoundTripper so the upload path+// can be driven without a network. Unlike a real server it always reads+// the whole request body and answers, which is exactly the misbehaviour+// (accepting a short body as a complete upload) these tests guard against.+type rtFunc func(*http.Request) (*http.Response, error)++func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }++func jsonResp(status int, body string) *http.Response {+ return &http.Response{+ StatusCode: status,+ Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),+ Header: http.Header{"Content-Type": {"application/json"}},+ Body: io.NopCloser(strings.NewReader(body)),+ ContentLength: int64(len(body)),+ }+}++func newFakeFs(rt http.RoundTripper) *Fs {+ ctx := context.Background()+ f := &Fs{+ name: "test",+ opt: Options{WorkspaceID: "ws_test"},+ pacer: fs.NewPacer(ctx, pacer.NewDefault(+ pacer.MinSleep(time.Millisecond),+ pacer.MaxSleep(time.Millisecond),+ pacer.DecayConstant(decayConstant),+ pacer.AttackConstant(attackConstant),+ )),+ }+ f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)+ return f+}++// fakeUploadServer records what an upload sends and lets a test inject+// failures per request+type fakeUploadServer struct {+ t *testing.T+ mu sync.Mutex+ resumable string // JSON for the init "resumable" field, "" for a single PUT+ puts [][]byte // bodies of PUT /api/upload/<id>+ parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt+ completes int+ aborts int // DELETE /api/upload/<id>+ failFirst map[string]bool // path -> answer the first request with a 502+ seenBefore map[string]bool+}++func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {+ return &fakeUploadServer{+ t: t,+ resumable: resumable,+ parts: map[int][][]byte{},+ failFirst: map[string]bool{},+ seenBefore: map[string]bool{},+ }+}++func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ var body []byte+ if r.Body != nil {+ var err error+ body, err = io.ReadAll(r.Body)+ require.NoError(s.t, err)+ }+ path := r.URL.Path+ // Bodies are recorded for every attempt, including the ones answered+ // with a 502, so tests can see exactly what each attempt carried.+ fail := s.failFirst[path] && !s.seenBefore[path]+ s.seenBefore[path] = true+ bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)+ switch {+ case r.Method == "POST" && path == "/api/upload/init":+ if s.resumable != "" {+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil+ }+ return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil+ case r.Method == "PUT" && path == "/api/upload/upl_1":+ s.puts = append(s.puts, body)+ if fail {+ return bad, nil+ }+ return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil+ case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):+ var n int+ _, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)+ require.NoError(s.t, err)+ s.parts[n] = append(s.parts[n], body)+ if fail {+ return bad, nil+ }+ return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil+ case r.Method == "POST" && path == "/api/upload/upl_1/complete":+ s.completes+++ return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil+ case r.Method == "DELETE" && path == "/api/upload/upl_1":+ s.aborts+++ return jsonResp(http.StatusOK, `{"ok":true}`), nil+ }+ s.t.Fatalf("unexpected request %s %s", r.Method, path)+ return nil, nil+}++func TestUploadSmallSendsWholeBody(t *testing.T) {+ srv := newFakeUploadServer(t, "")+ f := newFakeFs(rtFunc(srv.roundTrip))+ content := strings.Repeat("a", 100)++ resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.NoError(t, err)+ assert.Equal(t, "fil_1", resp.File.ID)+ require.Len(t, srv.puts, 1)+ assert.Equal(t, content, string(srv.puts[0]))+}++func TestUploadSmallRejectsShortSource(t *testing.T) {+ // The source declares 100 bytes but only supplies 60. A server which+ // accepts that as a complete upload must not turn into a success.+ srv := newFakeUploadServer(t, "")+ f := newFakeFs(rtFunc(srv.roundTrip))++ _, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.Error(t, err)+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)+ assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")+}++func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {+ // A retried PUT would resend an already-consumed reader, i.e. an+ // empty body, which the server would store as an empty file. The+ // single PUT therefore must not be retried at the low level.+ srv := newFakeUploadServer(t, "")+ srv.failFirst["/api/upload/upl_1"] = true+ f := newFakeFs(rtFunc(srv.roundTrip))++ content := strings.Repeat("a", 100)+ _, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.Error(t, err)+ require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")+ assert.Equal(t, content, string(srv.puts[0]))+}++const threeParts = `{"part_size":4,"total_parts":3}`++func TestUploadMultipartAssemblesAllBytes(t *testing.T) {+ srv := newFakeUploadServer(t, threeParts)+ f := newFakeFs(rtFunc(srv.roundTrip))+ content := "0123456789"++ resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.NoError(t, err)+ assert.Equal(t, "fil_1", resp.File.ID)+ assert.Equal(t, 1, srv.completes)+ require.Len(t, srv.parts, 3)+ assert.Equal(t, "0123", string(srv.parts[1][0]))+ assert.Equal(t, "4567", string(srv.parts[2][0]))+ assert.Equal(t, "89", string(srv.parts[3][0]))+}++func TestUploadMultipartRejectsShortSource(t *testing.T) {+ // 10 bytes declared, 8 supplied: the last part must never be sent+ // and the upload must not be completed.+ srv := newFakeUploadServer(t, threeParts)+ f := newFakeFs(rtFunc(srv.roundTrip))++ _, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.Error(t, err)+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)+ assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")+ assert.Empty(t, srv.parts[3], "short final part must not be uploaded")+ assert.Equal(t, 0, srv.completes, "upload must not be completed")+}++func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {+ // A part which fails transiently must be resent with all its bytes,+ // not with whatever is left of a consumed reader.+ srv := newFakeUploadServer(t, threeParts)+ srv.failFirst["/api/upload/upl_1/part/2"] = true+ f := newFakeFs(rtFunc(srv.roundTrip))++ _, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)+ require.NoError(t, err)+ require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")+ assert.Equal(t, "4567", string(srv.parts[2][0]))+ assert.Equal(t, "4567", string(srv.parts[2][1]))+ assert.Equal(t, "89", string(srv.parts[3][0]))+ assert.Equal(t, 1, srv.completes)+}++// An upload that fails after init has to hand its session back: the session+// holds one of the workspace's upload slots for ten minutes otherwise, and+// rclone's high-level retry opens a new one rather than resuming it.+func TestUploadAbortsTheSessionOnFailure(t *testing.T) {+ for _, tc := range []struct {+ name string+ resumable string+ content string+ size int64+ }{+ {"small file short source", "", "0123", 10},+ {"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},+ } {+ t.Run(tc.name, func(t *testing.T) {+ srv := newFakeUploadServer(t, tc.resumable)+ f := newFakeFs(rtFunc(srv.roundTrip))++ _, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),+ "file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)++ require.Error(t, err)+ assert.Equal(t, 1, srv.aborts)+ })+ }+}++// A successful upload must not abort anything+func TestUploadDoesNotAbortOnSuccess(t *testing.T) {+ srv := newFakeUploadServer(t, "")+ f := newFakeFs(rtFunc(srv.roundTrip))++ _, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),+ "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)++ require.NoError(t, err)+ assert.Equal(t, 0, srv.aborts)+}diff --git a/backend/dosya/dosya_workspace_hint_test.go b/backend/dosya/dosya_workspace_hint_test.gonew file mode 100644index 000000000..193f7b0c9--- /dev/null+++ b/backend/dosya/dosya_workspace_hint_test.go@@ -0,0 +1,73 @@+package dosya++import (+ "context"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// Every by-id call names the workspace, so the API can find an id it minted+// moments ago before its file_home index row lands.+func TestByIDCallsSendWorkspaceHint(t *testing.T) {+ seen := map[string]string{}+ f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {+ seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")+ switch r.URL.Path {+ case "/api/folders/fld_1":+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil+ case "/api/files/fil_1":+ return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil+ case "/api/folders/fld_1/move":+ return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil+ default:+ return jsonResp(http.StatusOK, `{"ok":true}`), nil+ }+ }))+ ctx := context.Background()++ require.NoError(t, f.removeFolder(ctx, "fld_1", false))+ require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))+ _, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")+ require.NoError(t, err)+ require.NoError(t, f.deleteFile(ctx, "fil_1"))+ require.NoError(t, f.renameFile(ctx, "fil_1", "n"))+ require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))+ _, err = f.copyFile(ctx, "fil_1", "fld_2", "n")+ require.NoError(t, err)+ _, err = f.createShareLink(ctx, "fil_1", 0)+ require.NoError(t, err)++ for call, header := range seen {+ assert.Equal(t, "ws_test", header, call)+ }+ assert.Len(t, seen, 8)+}++// The download's API request carries the hint; the presigned R2 fetch must not.+func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {+ var apiHint, blobHint string+ var srv *httptest.Server+ srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/api/files/fil_1/download" {+ apiHint = r.Header.Get("X-Dosya-Workspace")+ http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)+ return+ }+ blobHint = r.Header.Get("X-Dosya-Workspace")+ _, _ = w.Write([]byte("bytes"))+ }))+ defer srv.Close()+ f := newTestFs(srv)+ f.opt.APIURL = srv.URL++ rc, err := f.downloadFile(context.Background(), "fil_1", nil)+ require.NoError(t, err)+ _ = rc.Close()++ assert.Equal(t, "ws_test", apiHint)+ assert.Equal(t, "", blobHint)+}diff --git a/backend/dosya/dosya_workspace_test.go b/backend/dosya/dosya_workspace_test.gonew file mode 100644index 000000000..7d04a09db--- /dev/null+++ b/backend/dosya/dosya_workspace_test.go@@ -0,0 +1,52 @@+package dosya++import (+ "context"+ "net/http"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// fakeWorkspaceServer answers GET /api/workspaces with the given body+func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {+ return func(r *http.Request) (*http.Response, error) {+ require.Equal(t, "/api/workspaces", r.URL.Path)+ *calls+++ return jsonResp(http.StatusOK, body), nil+ }+}++func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {+ calls := 0+ f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))+ f.opt.WorkspaceID = ""++ require.NoError(t, f.findWorkspace(context.Background()))+ assert.Equal(t, "ws_only", f.opt.WorkspaceID)+ assert.Equal(t, 1, calls)+}++func TestFindWorkspaceListsTheChoices(t *testing.T) {+ calls := 0+ f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`++ `{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))+ f.opt.WorkspaceID = ""++ err := f.findWorkspace(context.Background())+ require.Error(t, err)+ // the ids have to be in the message: they are the only place the user+ // can read them, which is the whole point of listing them+ assert.Contains(t, err.Error(), "ws_one (One)")+ assert.Contains(t, err.Error(), "ws_two (Two)")+ assert.Empty(t, f.opt.WorkspaceID)+}++func TestFindWorkspaceWithNone(t *testing.T) {+ calls := 0+ f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))+ f.opt.WorkspaceID = ""++ require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")+}diff --git a/bin/make_manual.py b/bin/make_manual.pyindex 2ba0380b1..5e514cbbe 100755--- a/bin/make_manual.py+++ b/bin/make_manual.py@@ -44,6 +44,7 @@ docs = ["compress.md","combine.md","doi.md",+ "dosya.md","drime.md","dropbox.md","filefabric.md",diff --git a/docs/content/_index.md b/docs/content/_index.mdindex 0e357f221..56373921e 100644--- a/docs/content/_index.md+++ b/docs/content/_index.md@@ -128,6 +128,7 @@ WebDAV or S3, that work out of the box.){{< provider name="Cubbit DS3" home="https://cubbit.io/ds3-cloud" config="/s3/#Cubbit" >}}{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}+{{< provider name="dosya.dev" home="https://dosya.dev/" config="/dosya/" >}}{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}{{< provider name="Drime" home="https://www.drime.cloud/" config="/drime/" >}}{{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}diff --git a/docs/content/docs.md b/docs/content/docs.mdindex e06a54aad..e0cd0b74b 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -43,6 +43,7 @@ See the following for detailed instructions for- [Crypt](/crypt/) - to encrypt other remotes- [DigitalOcean Spaces](/s3/#digitalocean-spaces)- [Digi Storage](/koofr/#digi-storage)+- [dosya.dev](/dosya/)- [Drime](/drime/)- [Dropbox](/dropbox/)- [Enterprise File Fabric](/filefabric/)diff --git a/docs/content/dosya.md b/docs/content/dosya.mdnew file mode 100644index 000000000..5f2c399de--- /dev/null+++ b/docs/content/dosya.md@@ -0,0 +1,286 @@+---+title: "dosya.dev"+description: "Rclone docs for dosya.dev"+versionIntroduced: "v1.76"+---++# dosya.dev++[dosya.dev](https://dosya.dev) is a cloud file storage, sharing, and+synchronization platform with workspace-based multi-tenancy, file+versioning, and team collaboration features.++## Configuration++Here is an example of how to make a remote called `remote`. First run:++```console+rclone config+```++This will guide you through an interactive setup process:++```text+No remotes found, make a new one?+n) New remote+s) Set configuration password+q) Quit config+n/s/q> n++Enter name for new remote.+name> remote++Option Storage.+Type of storage to configure.+Choose a number from below, or type in your own value.+XX / dosya.dev+ \ (dosya)+Storage> dosya++Option api_key.+Your API Key, get it from https://dosya.dev/profile under "API keys".+Enter a value.+api_key> dos_YOURAPIKEY++Option workspace_id.+Your workspace ID.++Leave this blank to use the one workspace the API key can reach, which+is what a key restricted to a single workspace does. If the key can+reach several, rclone will refuse and list their IDs for you to choose+from.+Enter a value. Press Enter to leave empty.+workspace_id>++Edit advanced config?+y) Yes+n) No (default)+y/n> n++Configuration complete.+Options:+- type: dosya+- api_key: dos_YOURAPIKEY+Keep this "remote" remote?+y) Yes this is OK (default)+e) Edit this remote+d) Delete this remote+y/e/d> y+```++Once configured you can then use `rclone` like this (replace `remote`+with the name you gave your remote):++List directories and files in the top level of your dosya workspace++```console+rclone lsf remote:+```++To copy a local directory to a dosya directory called backup++```console+rclone copy /home/source remote:backup+```++The dosya backend uploads large files in parts and stores them as single+files that remain accessible through the web app and other clients. If you+previously used a WebDAV remote with a chunker overlay for large uploads,+you can use this backend without the overlay.++### Getting your API key++1. Log in to [dosya.dev](https://dosya.dev)+2. Go to [Profile](https://dosya.dev/profile) > **API keys**+3. Create a new API key+4. Copy the key (starts with `dos_`)++A key can be restricted to a single workspace when you create it, which+is the simplest thing to do for rclone: `workspace_id` can then be left+out of the config entirely.++### Workspaces++Leave `workspace_id` blank and rclone asks which workspaces the key can+reach. A key restricted to one workspace, and an account that has only+one, both resolve without any configuration. If the key can reach+several, rclone refuses and lists their IDs so you can set the one you+want:++```text+this API key can reach 2 workspaces, so workspace_id must be set to one of:+ ws_AAAAAAAA (Work)+ ws_BBBBBBBB (Personal)+```++To access several workspaces at once, create a separate remote for each:++```console+rclone config create work dosya api_key=dos_XXX workspace_id=ws_WORK+rclone config create personal dosya api_key=dos_XXX workspace_id=ws_PERSONAL+```++Then use them independently:++```console+rclone ls work:+rclone copy work:docs/ personal:backup/+```++### Modification times and hashes++dosya.dev supports modification times with a resolution of 1 second. The+modification time is sent with the upload, so a copied or synced file keeps+its original timestamp. Modification times cannot be changed after upload+without re-uploading the file.++dosya.dev supports SHA-256 checksums. The server computes the hash of a+single-shot (small file) upload from the bytes it receives, so `rclone check`+and hash-based sync work without downloading. Large files uploaded in multiple+parts have no server-side hash (the parts are never re-read to hash the whole+object), the same limitation the S3 backend has for multipart objects; those+objects report an empty hash.++### Restricted filename characters++In addition to the [default restricted characters set](/overview/#restricted-characters)+the following characters are also replaced:++| Character | Value | Replacement |+| --------- |:-----:|:-----------:|+| " | 0x22 | " |+| ' | 0x27 | ' |+| < | 0x3C | < |+| > | 0x3E | > |+| \ | 0x5C | \ |++Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),+as they can't be used in JSON strings.++### Public Links++dosya.dev supports `rclone link` to create public share links to+files. You can use the `--expire` flag to specify how long the link+should be valid.++```console+rclone link remote:path/to/file.txt+rclone link --expire 7d remote:path/to/file.txt+```++<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/dosya/dosya.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->+### Standard options++Here are the Standard options specific to dosya (dosya.dev).++#### --dosya-api-key++Your API Key, get it from https://dosya.dev/profile under "API keys".++Properties:++- Config: api_key+- Env Var: RCLONE_DOSYA_API_KEY+- Type: string+- Required: true++#### --dosya-workspace-id++Your workspace ID.++Leave this blank to use the one workspace the API key can reach, which+is what a key restricted to a single workspace does. If the key can+reach several, rclone will refuse and list their IDs for you to choose+from.++Properties:++- Config: workspace_id+- Env Var: RCLONE_DOSYA_WORKSPACE_ID+- Type: string+- Required: false++### Advanced options++Here are the Advanced options specific to dosya (dosya.dev).++#### --dosya-api-url++API base URL.++Properties:++- Config: api_url+- Env Var: RCLONE_DOSYA_API_URL+- Type: string+- Default: "https://api.dosya.dev"++#### --dosya-upload-concurrency++Concurrency for multipart uploads.++This is the number of parts of the same file that are uploaded+concurrently. The first part is always uploaded on its own, because+the server creates the upload when it receives it.++Each part in flight is held in memory, so memory use grows with+this value times the part size the server chooses.++If you are uploading large files over high-speed links and these+uploads do not fully utilize your bandwidth, then increasing this+may help to speed up the transfers.++Properties:++- Config: upload_concurrency+- Env Var: RCLONE_DOSYA_UPLOAD_CONCURRENCY+- Type: int+- Default: 4++#### --dosya-encoding++The encoding for the backend.++See the [encoding section in the overview](/overview/#encoding) for more info.++Properties:++- Config: encoding+- Env Var: RCLONE_DOSYA_ENCODING+- Type: Encoding+- Default: Slash,LtGt,DoubleQuote,SingleQuote,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot++#### --dosya-description++Description of the remote.++Properties:++- Config: description+- Env Var: RCLONE_DOSYA_DESCRIPTION+- Type: string+- Required: false++<!-- autogenerated options stop -->++## Limitations++dosya.dev only supports filenames up to 255 characters in length.++Empty files (0 bytes) cannot be uploaded to dosya.dev.++Upload requires knowing the file size in advance, so streaming+uploads of unknown-size files are not supported.++Each workspace limits the number of uploads a user can have in flight+at once (5 by default, configurable in the workspace settings). When+`--transfers` is set higher than that limit, rclone waits for a free+upload slot rather than failing the transfer, retrying every 5 seconds+for up to 10 minutes.++### Deleting files and directories++rclone deletes files and directories permanently, in a single request+(`DELETE ?permanent=1`), bypassing the workspace trash. Removing a+directory also asks the server whether it is empty (`?if_empty=1`)+rather than listing it first.diff --git a/docs/data/backends/dosya.yaml b/docs/data/backends/dosya.yamlnew file mode 100644index 000000000..c12c337c1--- /dev/null+++ b/docs/data/backends/dosya.yaml@@ -0,0 +1,26 @@+backend: dosya+name: dosya.dev+tier: Tier 1+maintainers: External+features_score: 5+integration_tests: Passing+data_integrity: Hash+performance: Medium+adoption: New+docs: Full+security: High+virtual: false+remote: 'TestDosya:'+features:+- About+- CanHaveEmptyDirectories+- Copy+- DirCacheFlush+- DirMove+- Move+- PublicLink+- Purge+- ReadMimeType+hashes:+- sha256+precision: 1000000000diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.htmlindex b7f80d285..cff6fa02d 100644--- a/docs/layouts/chrome/navbar.html+++ b/docs/layouts/chrome/navbar.html@@ -70,6 +70,7 @@<a class="dropdown-item" href="/crypt/">Crypt (encrypts the others)</a><span class="dropdown-letter-heading">D – F</span><a class="dropdown-item" href="/koofr/#digi-storage">Digi Storage</a>+ <a class="dropdown-item" href="/dosya/">dosya.dev</a><a class="dropdown-item" href="/drime/">Drime</a><a class="dropdown-item" href="/dropbox/">Dropbox</a><a class="dropdown-item" href="/filefabric/">Enterprise File Fabric</a>diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yamlindex 81425dcb6..3cb9af961 100644--- a/fstest/test_all/config.yaml+++ b/fstest/test_all/config.yaml@@ -723,6 +723,14 @@ backends:listretries: 5ignore:- TestRWFileHandleWriteNoWrite+ - backend: "dosya"+ remote: "TestDosya:"+ # Every upload is a metadata round trip of several seconds regardless+ # of size, so fs/sync (TestTransform alone is 11 x ~3.5 min) needs+ # well over the default 1h budget.+ extratime: 3.0+ ignore:+ - TestRWFileHandleWriteNoWrite- backend: "drime"remote: "TestDrime:"ignoretests: