"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
test speed: add command to test a specified remotes speed
dougal99e8a63df252
diff --git a/cmd/test/makefiles/makefiles.go b/cmd/test/makefiles/makefiles.goindex 40390a663..921d708cf 100644--- a/cmd/test/makefiles/makefiles.go+++ b/cmd/test/makefiles/makefiles.go@@ -61,15 +61,18 @@ func init() {test.Command.AddCommand(makefileCmd)makefileFlags := makefileCmd.Flags()- // Common flags to makefiles and makefile- for _, f := range []*pflag.FlagSet{makefilesFlags, makefileFlags} {- flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "")- flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "")- flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "")- flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "")- flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "")- flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "")- }+ addCommonFlags(makefilesFlags)+ addCommonFlags(makefileFlags)+}++// Common flags for makefiles and makefile+func addCommonFlags(f *pflag.FlagSet) {+ flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "")+ flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "")+ flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "")+ flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "")+ flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "")+ flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "")}var makefilesCmd = &cobra.Command{@@ -123,20 +126,24 @@ var makefileCmd = &cobra.Command{if err != nil {fs.Fatalf(nil, "Failed to parse size %q: %v", args[0], err)}- start := time.Now()- fs.Logf(nil, "Creating %d files of size %v.", len(args[1:]), size)- totalBytes := int64(0)- for _, filePath := range args[1:] {- dir := filepath.Dir(filePath)- name := filepath.Base(filePath)- writeFile(dir, name, int64(size))- totalBytes += int64(size)- }- dt := time.Since(start)- fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))+ makefiles(size, args[1:])},}+func makefiles(size fs.SizeSuffix, files []string) {+ start := time.Now()+ fs.Logf(nil, "Creating %d files of size %v.", len(files), size)+ totalBytes := int64(0)+ for _, filePath := range files {+ dir := filepath.Dir(filePath)+ name := filepath.Base(filePath)+ writeFile(dir, name, int64(size))+ totalBytes += int64(size)+ }+ dt := time.Since(start)+ fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))+}+func bool2int(b bool) int {if b {return 1diff --git a/cmd/test/makefiles/speed.go b/cmd/test/makefiles/speed.gonew file mode 100644index 000000000..fc7edeb6d--- /dev/null+++ b/cmd/test/makefiles/speed.go@@ -0,0 +1,235 @@+package makefiles++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "os"+ "path"+ "time"++ "github.com/rclone/rclone/cmd"+ "github.com/rclone/rclone/cmd/test"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/cache"+ "github.com/rclone/rclone/fs/config/flags"+ "github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/fs/sync"+ "github.com/rclone/rclone/lib/atexit"+ "github.com/rclone/rclone/lib/random"+ "github.com/spf13/cobra"+)++var (+ // Flags+ testTime = fs.Duration(15 * time.Second)+ fcap = 100+ small = fs.SizeSuffix(1024)+ medium = fs.SizeSuffix(10 * 1024 * 1024)+ large = fs.SizeSuffix(1024 * 1024 * 1024)+ useJSON = false+)++func init() {+ test.Command.AddCommand(speedCmd)++ speedFlags := speedCmd.Flags()+ flags.FVarP(speedFlags, &testTime, "test-time", "", "Length for each test to run", "")+ flags.IntVarP(speedFlags, &fcap, "file-cap", "", fcap, "Maximum number of files to use in each test", "")+ flags.FVarP(speedFlags, &small, "small", "", "Size of small files", "")+ flags.FVarP(speedFlags, &medium, "medium", "", "Size of medium files", "")+ flags.FVarP(speedFlags, &large, "large", "", "Size of large files", "")+ flags.BoolVarP(speedFlags, &useJSON, "json", "", useJSON, "Output only results in JSON format", "")++ addCommonFlags(speedFlags)+}++func logf(text string, args ...any) {+ if !useJSON {+ fmt.Printf(text, args...)+ }+}++var speedCmd = &cobra.Command{+ Use: "speed <remote> [flags]",+ Short: `Run a speed test to the remote`,+ Long: `Run a speed test to the remote.++ This command runs a series of uploads and downloads to the remote, measuring+ and printing the speed of each test using varying file sizes and numbers of+ files.++ Test time can be innaccurate with small file caps and large files. As it+ uses the results of an initial test to determine how many files to use in+ each subsequent test.++ It is recommended to use -q flag for a simpler output. e.g.:++ rlone test speed remote: -q++ **NB** This command will create and delete files on the remote in a randomly+ named directory which should be tidied up after.++ You can use the --json flag to only print the results in JSON format.`,+ Annotations: map[string]string{+ "versionIntroduced": "v1.72",+ },+ RunE: func(command *cobra.Command, args []string) error {+ ctx := command.Context()+ cmd.CheckArgs(1, 1, command, args)+ commonInit()++ // initial test+ size := fs.SizeSuffix(1024 * 1024)+ logf("Running initial test for 4 files of size %v\n", size)+ stats, err := speedTest(ctx, 4, size, args[0])+ if err != nil {+ return fmt.Errorf("speed test failed: %w", err)+ }++ var results []*Stats++ // main tests+ logf("\nTest Time: %v, File cap: %d\n", testTime, fcap)+ for _, size := range []fs.SizeSuffix{small, medium, large} {+ numberOfFilesUpload := int((float64(stats.Upload.Speed) * time.Duration(testTime).Seconds()) / float64(size))+ numberOfFilesDownload := int((float64(stats.Download.Speed) * time.Duration(testTime).Seconds()) / float64(size))+ numberOfFiles := min(numberOfFilesUpload, numberOfFilesDownload)++ logf("\nNumber of files for upload and download: %v\n", numberOfFiles)+ if numberOfFiles < 1 {+ logf("Skipping test for file size %v as calculated number of files is 0\n", size)+ continue+ } else if numberOfFiles > fcap {+ numberOfFiles = fcap+ logf("Capping test for file size %v to %v files\n", size, fcap)+ }++ logf("Running test for %d files of size %v\n", numberOfFiles, size)+ s, err := speedTest(ctx, numberOfFiles, size, args[0])+ if err != nil {+ return fmt.Errorf("speed test failed: %w", err)+ }+ results = append(results, s)++ }++ if useJSON {+ b, err := json.MarshalIndent(results, "", " ")+ if err != nil {+ return fmt.Errorf("failed to marshal results to JSON: %w", err)+ }+ fmt.Println(string(b))+ }++ return nil+ },+}++// Stats of a speed test+type Stats struct {+ Size fs.SizeSuffix+ NumberOfFiles int+ Upload TestResult+ Download TestResult+}++// TestResult of a speed test operation+type TestResult struct {+ Bytes int64+ Duration time.Duration+ Speed fs.SizeSuffix+}++// measures stats for speedTest operations+func measure(desc string, f func() error, size fs.SizeSuffix, numberOfFiles int, tr *TestResult) error {+ start := time.Now()+ err := f()+ dt := time.Since(start)+ if err != nil {+ return err+ }+ tr.Duration = dt+ tr.Bytes = int64(size) * int64(numberOfFiles)+ tr.Speed = fs.SizeSuffix(float64(tr.Bytes) / dt.Seconds())+ logf("%-20s: %vB in %v at %vB/s\n", desc, tr.Bytes, dt.Round(time.Millisecond), tr.Speed)+ return err+}++func speedTest(ctx context.Context, numberOfFiles int, size fs.SizeSuffix, remote string) (*Stats, error) {+ stats := Stats{+ Size: size,+ NumberOfFiles: numberOfFiles,+ }++ tempDirName := "rclone-speed-test-" + random.String(8)+ tempDirPath := path.Join(remote, tempDirName)+ fremote := cmd.NewFsDir([]string{tempDirPath})+ aErr := io.EOF+ defer atexit.OnError(&aErr, func() {+ err := operations.Purge(ctx, fremote, "")+ if err != nil {+ fs.Debugf(fremote, "Failed to remove temp dir %q: %v", tempDirPath, err)+ }+ })()++ flocalDir, err := os.MkdirTemp("", "rclone-speedtest-local-")+ if err != nil {+ return nil, fmt.Errorf("failed to create local temp dir: %w", err)+ }+ defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(flocalDir) })()++ flocal, err := cache.Get(ctx, flocalDir)+ if err != nil {+ return nil, fmt.Errorf("failed to create local fs: %w", err)+ }++ fdownloadDir, err := os.MkdirTemp("", "rclone-speedtest-download-")+ if err != nil {+ return nil, fmt.Errorf("failed to create download temp dir: %w", err)+ }+ defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(fdownloadDir) })()++ fdownload, err := cache.Get(ctx, fdownloadDir)+ if err != nil {+ return nil, fmt.Errorf("failed to create download fs: %w", err)+ }++ // make the largest amount of files we will need+ files := make([]string, numberOfFiles)+ for i := range files {+ files[i] = path.Join(flocalDir, fmt.Sprintf("file%03d-%v.bin", i, size))+ }+ makefiles(size, files)++ // upload files+ err = measure("Upload", func() error {+ return sync.CopyDir(ctx, fremote, flocal, false)+ }, size, numberOfFiles, &stats.Upload)+ if err != nil {+ return nil, fmt.Errorf("failed to Copy to remote: %w", err)+ }++ // download files+ err = measure("Download", func() error {+ return sync.CopyDir(ctx, fdownload, fremote, false)+ }, size, numberOfFiles, &stats.Download)+ if err != nil {+ return nil, fmt.Errorf("failed to Copy from remote: %w", err)+ }++ // check files+ opt := operations.CheckOpt{+ Fsrc: flocal,+ Fdst: fdownload,+ OneWay: false,+ }+ logf("Checking file integrity\n")+ err = operations.CheckDownload(ctx, &opt)+ if err != nil {+ return nil, fmt.Errorf("failed to check redownloaded files were identical: %w", err)+ }++ return &stats, nil+}