"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
package drime
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/backend/drime/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
testPerPage = 5 // entries per page the test server returns
testLimit = 20 // entries the test server will page through
)
// listingServer serves /drive/file-entries from entries the way Drime
// does.
//
// It returns testPerPage entries per page and won't page past testLimit
// entries. Asking for a later page returns the last page it allows
// again, with last_page one beyond it.
//
// It counts the requests made in requests.
func listingServer(t *testing.T, entries []api.Item, requests *atomic.Int64) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
assert.Equal(t, "/drive/file-entries", r.URL.Path)
params := r.URL.Query()
page, err := strconv.Atoi(params.Get("page"))
require.NoError(t, err)
maxPage := testLimit / testPerPage
currentPage := min(page, maxPage)
start := min((currentPage-1)*testPerPage, len(entries))
end := min(start+testPerPage, len(entries))
lastPage := currentPage
if page > maxPage || end < len(entries) {
lastPage = currentPage + 1
}
w.Header().Set("Content-Type", "application/json")
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
"current_page": currentPage,
"last_page": lastPage,
"per_page": testPerPage,
"total": (currentPage + 1) * testPerPage,
"data": entries[start:end],
}))
}))
}
// makeEntries makes n file entries
func makeEntries(n int) []api.Item {
entries := make([]api.Item, n)
for i := range entries {
entries[i] = api.Item{
ID: json.Number(strconv.Itoa(1000 + i)),
Name: fmt.Sprintf("file%05d", i),
Type: "text",
}
}
return entries
}
// testFs makes an Fs which uses server as its API
func testFs(t *testing.T, server *httptest.Server) *Fs {
t.Helper()
f := &Fs{
opt: Options{ListChunk: 1000},
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
}
f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
f.srv.SetErrorHandler(errorHandler)
return f
}
// listNames lists a directory with listAll and returns the names found
func listNames(ctx context.Context, f *Fs) (names []string, err error) {
_, err = f.listAll(ctx, "1", false, false, "", func(item *api.Item) bool {
names = append(names, item.Name)
return false
})
return names, err
}
// TestListAll checks that listAll returns every entry exactly once
func TestListAll(t *testing.T) {
for _, test := range []struct {
name string
entries int
}{
{name: "Empty", entries: 0},
{name: "OnePage", entries: 3},
{name: "ExactPages", entries: 10},
{name: "PartialLastPage", entries: 13},
{name: "AtLimit", entries: testLimit},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var requests atomic.Int64
entries := makeEntries(test.entries)
server := listingServer(t, entries, &requests)
defer server.Close()
f := testFs(t, server)
names, err := listNames(ctx, f)
require.NoError(t, err)
var want []string
for _, entry := range entries {
want = append(want, entry.Name)
}
assert.ElementsMatch(t, want, names)
})
}
}
// TestListAllTooManyEntries checks that listAll returns an error for a
// directory with more entries than the server will list, rather than
// looping forever.
func TestListAllTooManyEntries(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var requests atomic.Int64
server := listingServer(t, makeEntries(2*testLimit), &requests)
defer server.Close()
f := testFs(t, server)
_, err := listNames(ctx, f)
require.Error(t, err)
assert.NoError(t, ctx.Err(), "listing didn't finish")
assert.LessOrEqual(t, requests.Load(), int64(testLimit/testPerPage+1))
}
// TestGetItemListError checks that getItem returns listing errors
// rather than fs.ErrorObjectNotFound.
func TestGetItemListError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"message":"access denied"}`, http.StatusForbidden)
}))
defer server.Close()
f := testFs(t, server)
_, err := f.getItem(context.Background(), "1000", "1", "file00000")
require.Error(t, err)
assert.NotErrorIs(t, err, fs.ErrorObjectNotFound)
assert.Contains(t, err.Error(), "access denied")
}