"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
package dosya
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
status int
header string
want time.Duration
ok bool
}{
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
{"missing header", http.StatusServiceUnavailable, "", 0, false},
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
} {
t.Run(tc.name, func(t *testing.T) {
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
if tc.header != "" {
resp.Header.Set("Retry-After", tc.header)
}
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
assert.True(t, retry)
d, ok := pacer.IsRetryAfter(err)
assert.Equal(t, tc.ok, ok)
if tc.ok {
assert.Equal(t, tc.want, d)
}
})
}
}
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
resp.Header.Set("Retry-After", "30")
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
assert.False(t, retry)
_, ok := pacer.IsRetryAfter(err)
assert.False(t, ok)
}
func TestRemoveFolderPurgesPermanently(t *testing.T) {
// a big subtree purges in bounded passes: 202 complete=false repeats
// the same request until complete=true
answers := []struct {
status int
body string
}{
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
}
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
require.Equal(t, "DELETE", r.Method)
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
a := answers[calls]
calls++
return jsonResp(a.status, a.body), nil
}))
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
assert.Equal(t, len(answers), calls)
}
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
}))
err := f.removeFolder(context.Background(), "fld_1", false)
require.Error(t, err)
assert.Contains(t, err.Error(), "locked file")
assert.Equal(t, 1, calls)
}
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
}))
err := f.removeFolder(context.Background(), "fld_1", true)
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
}
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
}))
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
assert.Equal(t, 1, calls)
}
func TestRmdirMakesOneRequest(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "DELETE", r.Method)
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
}))
f.dirCache = dircache.New("", rootID, f)
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
f.dirCache.Put("sub", "fld_1")
require.NoError(t, f.Rmdir(context.Background(), "sub"))
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
}
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
}))
f.dirCache = dircache.New("", rootID, f)
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
f.dirCache.Put("sub", "fld_1")
err := f.Rmdir(context.Background(), "sub")
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
}
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
for _, tc := range []struct {
name string
permanent bool
wantCalls int
}{
{"today's API purges in one call", true, 1},
{"an older server ignores the flag and still gets a second DELETE", false, 2},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "DELETE", r.Method)
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
}))
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
assert.Equal(t, tc.wantCalls, calls)
})
}
}
// decodeBody reads a JSON request body into a map
func decodeBody(t *testing.T, r *http.Request) map[string]any {
b, err := io.ReadAll(r.Body)
require.NoError(t, err)
var m map[string]any
require.NoError(t, json.Unmarshal(b, &m))
return m
}
func TestMoveFolderSendsName(t *testing.T) {
for _, tc := range []struct {
name string
newName string
respBody string
wantRenamed bool
}{
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
{"older server ignores name", "dst", `{"ok":true}`, false},
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
} {
t.Run(tc.name, func(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
body := decodeBody(t, r)
assert.Equal(t, "fld_parent", body["parent_id"])
if tc.newName == "" {
assert.NotContains(t, body, "name")
} else {
assert.Equal(t, tc.newName, body["name"])
}
return jsonResp(http.StatusOK, tc.respBody), nil
}))
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
require.NoError(t, err)
assert.Equal(t, tc.wantRenamed, renamed)
})
}
}
func TestCopyFileSendsName(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
body := decodeBody(t, r)
assert.Equal(t, "fld_dst", body["folder_id"])
assert.Equal(t, "final.txt", body["name"])
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
}))
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
require.NoError(t, err)
assert.Equal(t, "fil_2", resp.FileID)
assert.Equal(t, "final.txt", resp.Name)
}