Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

test speed: add command to test a specified remotes speed

dougal99e8a63df252
diff --git a/cmd/test/makefiles/makefiles.go b/cmd/test/makefiles/makefiles.go
index 40390a663..921d708cf 100644
--- a/cmd/test/makefiles/makefiles.go
+++ b/cmd/test/makefiles/makefiles.go
@@ -61,15 +61,18 @@ func init() {
test.Command.AddCommand(makefileCmd)
makefileFlags := makefileCmd.Flags()
- // Common flags to makefiles and makefile
- for _, f := range []*pflag.FlagSet{makefilesFlags, makefileFlags} {
- flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "")
- flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "")
- flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "")
- flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "")
- flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "")
- flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "")
- }
+ addCommonFlags(makefilesFlags)
+ addCommonFlags(makefileFlags)
+}
+
+// Common flags for makefiles and makefile
+func addCommonFlags(f *pflag.FlagSet) {
+ flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "")
+ flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "")
+ flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "")
+ flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "")
+ flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "")
+ flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "")
}
var makefilesCmd = &cobra.Command{
@@ -123,20 +126,24 @@ var makefileCmd = &cobra.Command{
if err != nil {
fs.Fatalf(nil, "Failed to parse size %q: %v", args[0], err)
}
- start := time.Now()
- fs.Logf(nil, "Creating %d files of size %v.", len(args[1:]), size)
- totalBytes := int64(0)
- for _, filePath := range args[1:] {
- dir := filepath.Dir(filePath)
- name := filepath.Base(filePath)
- writeFile(dir, name, int64(size))
- totalBytes += int64(size)
- }
- dt := time.Since(start)
- fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
+ makefiles(size, args[1:])
},
}
+func makefiles(size fs.SizeSuffix, files []string) {
+ start := time.Now()
+ fs.Logf(nil, "Creating %d files of size %v.", len(files), size)
+ totalBytes := int64(0)
+ for _, filePath := range files {
+ dir := filepath.Dir(filePath)
+ name := filepath.Base(filePath)
+ writeFile(dir, name, int64(size))
+ totalBytes += int64(size)
+ }
+ dt := time.Since(start)
+ fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
+}
+
func bool2int(b bool) int {
if b {
return 1
diff --git a/cmd/test/makefiles/speed.go b/cmd/test/makefiles/speed.go
new file mode 100644
index 000000000..fc7edeb6d
--- /dev/null
+++ b/cmd/test/makefiles/speed.go
@@ -0,0 +1,235 @@
+package makefiles
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "time"
+
+ "github.com/rclone/rclone/cmd"
+ "github.com/rclone/rclone/cmd/test"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/config/flags"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/sync"
+ "github.com/rclone/rclone/lib/atexit"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/spf13/cobra"
+)
+
+var (
+ // Flags
+ testTime = fs.Duration(15 * time.Second)
+ fcap = 100
+ small = fs.SizeSuffix(1024)
+ medium = fs.SizeSuffix(10 * 1024 * 1024)
+ large = fs.SizeSuffix(1024 * 1024 * 1024)
+ useJSON = false
+)
+
+func init() {
+ test.Command.AddCommand(speedCmd)
+
+ speedFlags := speedCmd.Flags()
+ flags.FVarP(speedFlags, &testTime, "test-time", "", "Length for each test to run", "")
+ flags.IntVarP(speedFlags, &fcap, "file-cap", "", fcap, "Maximum number of files to use in each test", "")
+ flags.FVarP(speedFlags, &small, "small", "", "Size of small files", "")
+ flags.FVarP(speedFlags, &medium, "medium", "", "Size of medium files", "")
+ flags.FVarP(speedFlags, &large, "large", "", "Size of large files", "")
+ flags.BoolVarP(speedFlags, &useJSON, "json", "", useJSON, "Output only results in JSON format", "")
+
+ addCommonFlags(speedFlags)
+}
+
+func logf(text string, args ...any) {
+ if !useJSON {
+ fmt.Printf(text, args...)
+ }
+}
+
+var speedCmd = &cobra.Command{
+ Use: "speed <remote> [flags]",
+ Short: `Run a speed test to the remote`,
+ Long: `Run a speed test to the remote.
+
+ This command runs a series of uploads and downloads to the remote, measuring
+ and printing the speed of each test using varying file sizes and numbers of
+ files.
+
+ Test time can be innaccurate with small file caps and large files. As it
+ uses the results of an initial test to determine how many files to use in
+ each subsequent test.
+
+ It is recommended to use -q flag for a simpler output. e.g.:
+
+ rlone test speed remote: -q
+
+ **NB** This command will create and delete files on the remote in a randomly
+ named directory which should be tidied up after.
+
+ You can use the --json flag to only print the results in JSON format.`,
+ Annotations: map[string]string{
+ "versionIntroduced": "v1.72",
+ },
+ RunE: func(command *cobra.Command, args []string) error {
+ ctx := command.Context()
+ cmd.CheckArgs(1, 1, command, args)
+ commonInit()
+
+ // initial test
+ size := fs.SizeSuffix(1024 * 1024)
+ logf("Running initial test for 4 files of size %v\n", size)
+ stats, err := speedTest(ctx, 4, size, args[0])
+ if err != nil {
+ return fmt.Errorf("speed test failed: %w", err)
+ }
+
+ var results []*Stats
+
+ // main tests
+ logf("\nTest Time: %v, File cap: %d\n", testTime, fcap)
+ for _, size := range []fs.SizeSuffix{small, medium, large} {
+ numberOfFilesUpload := int((float64(stats.Upload.Speed) * time.Duration(testTime).Seconds()) / float64(size))
+ numberOfFilesDownload := int((float64(stats.Download.Speed) * time.Duration(testTime).Seconds()) / float64(size))
+ numberOfFiles := min(numberOfFilesUpload, numberOfFilesDownload)
+
+ logf("\nNumber of files for upload and download: %v\n", numberOfFiles)
+ if numberOfFiles < 1 {
+ logf("Skipping test for file size %v as calculated number of files is 0\n", size)
+ continue
+ } else if numberOfFiles > fcap {
+ numberOfFiles = fcap
+ logf("Capping test for file size %v to %v files\n", size, fcap)
+ }
+
+ logf("Running test for %d files of size %v\n", numberOfFiles, size)
+ s, err := speedTest(ctx, numberOfFiles, size, args[0])
+ if err != nil {
+ return fmt.Errorf("speed test failed: %w", err)
+ }
+ results = append(results, s)
+
+ }
+
+ if useJSON {
+ b, err := json.MarshalIndent(results, "", " ")
+ if err != nil {
+ return fmt.Errorf("failed to marshal results to JSON: %w", err)
+ }
+ fmt.Println(string(b))
+ }
+
+ return nil
+ },
+}
+
+// Stats of a speed test
+type Stats struct {
+ Size fs.SizeSuffix
+ NumberOfFiles int
+ Upload TestResult
+ Download TestResult
+}
+
+// TestResult of a speed test operation
+type TestResult struct {
+ Bytes int64
+ Duration time.Duration
+ Speed fs.SizeSuffix
+}
+
+// measures stats for speedTest operations
+func measure(desc string, f func() error, size fs.SizeSuffix, numberOfFiles int, tr *TestResult) error {
+ start := time.Now()
+ err := f()
+ dt := time.Since(start)
+ if err != nil {
+ return err
+ }
+ tr.Duration = dt
+ tr.Bytes = int64(size) * int64(numberOfFiles)
+ tr.Speed = fs.SizeSuffix(float64(tr.Bytes) / dt.Seconds())
+ logf("%-20s: %vB in %v at %vB/s\n", desc, tr.Bytes, dt.Round(time.Millisecond), tr.Speed)
+ return err
+}
+
+func speedTest(ctx context.Context, numberOfFiles int, size fs.SizeSuffix, remote string) (*Stats, error) {
+ stats := Stats{
+ Size: size,
+ NumberOfFiles: numberOfFiles,
+ }
+
+ tempDirName := "rclone-speed-test-" + random.String(8)
+ tempDirPath := path.Join(remote, tempDirName)
+ fremote := cmd.NewFsDir([]string{tempDirPath})
+ aErr := io.EOF
+ defer atexit.OnError(&aErr, func() {
+ err := operations.Purge(ctx, fremote, "")
+ if err != nil {
+ fs.Debugf(fremote, "Failed to remove temp dir %q: %v", tempDirPath, err)
+ }
+ })()
+
+ flocalDir, err := os.MkdirTemp("", "rclone-speedtest-local-")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create local temp dir: %w", err)
+ }
+ defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(flocalDir) })()
+
+ flocal, err := cache.Get(ctx, flocalDir)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create local fs: %w", err)
+ }
+
+ fdownloadDir, err := os.MkdirTemp("", "rclone-speedtest-download-")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create download temp dir: %w", err)
+ }
+ defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(fdownloadDir) })()
+
+ fdownload, err := cache.Get(ctx, fdownloadDir)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create download fs: %w", err)
+ }
+
+ // make the largest amount of files we will need
+ files := make([]string, numberOfFiles)
+ for i := range files {
+ files[i] = path.Join(flocalDir, fmt.Sprintf("file%03d-%v.bin", i, size))
+ }
+ makefiles(size, files)
+
+ // upload files
+ err = measure("Upload", func() error {
+ return sync.CopyDir(ctx, fremote, flocal, false)
+ }, size, numberOfFiles, &stats.Upload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to Copy to remote: %w", err)
+ }
+
+ // download files
+ err = measure("Download", func() error {
+ return sync.CopyDir(ctx, fdownload, fremote, false)
+ }, size, numberOfFiles, &stats.Download)
+ if err != nil {
+ return nil, fmt.Errorf("failed to Copy from remote: %w", err)
+ }
+
+ // check files
+ opt := operations.CheckOpt{
+ Fsrc: flocal,
+ Fdst: fdownload,
+ OneWay: false,
+ }
+ logf("Checking file integrity\n")
+ err = operations.CheckDownload(ctx, &opt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check redownloaded files were identical: %w", err)
+ }
+
+ return &stats, nil
+}