"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: add operations/hashsum to the rc as rclone hashsum equivalent
Nick Craig-Woodd50572b1083b
diff --git a/fs/operations/rc.go b/fs/operations/rc.goindex 533c3601e..1435e66de 100644--- a/fs/operations/rc.go+++ b/fs/operations/rc.go@@ -876,3 +876,80 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {}return out, nil}++func init() {+ rc.Add(rc.Call{+ Path: "operations/hashsum",+ AuthRequired: true,+ Fn: rcHashsum,+ Title: "Produces a hashsum file for all the objects in the path.",+ Help: `Produces a hash file for all the objects in the path using the hash+named. The output is in the same format as the standard+md5sum/sha1sum tool.++This takes the following parameters:++- fs - a remote name string e.g. "drive:" for the source, "/" for local filesystem+ - this can point to a file and just that file will be returned in the listing.+- hashType - type of hash to be used+- download - check by downloading rather than with hash (boolean)+- base64 - output the hashes in base64 rather than hex (boolean)++If you supply the download flag, it will download the data from the+remote and create the hash on the fly. This can be useful for remotes+that don't support the given hash or if you really want to check all+the data.++Note that if you wish to supply a checkfile to check hashes against+the current files then you should use operations/check instead of+operations/hashsum.++Returns:++- hashsum - array of strings of the hashes+- hashType - type of hash used++Example:++ $ rclone rc --loopback operations/hashsum fs=bin hashType=MD5 download=true base64=true+ {+ "hashType": "md5",+ "hashsum": [+ "WTSVLpuiXyJO_kGzJerRLg== backend-versions.sh",+ "v1b_OlWCJO9LtNq3EIKkNQ== bisect-go-rclone.sh",+ "VHbmHzHh4taXzgag8BAIKQ== bisect-rclone.sh",+ ]+ }++See the [hashsum](/commands/rclone_hashsum/) command for more information on the above.+`,+ })+}++// Hashsum a directory+func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) {+ ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs")+ if err != nil {+ return nil, err+ }++ download, _ := in.GetBool("download")+ base64, _ := in.GetBool("base64")+ hashType, err := in.GetString("hashType")+ if err != nil {+ return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)+ }+ var ht hash.Type+ err = ht.Set(hashType)+ if err != nil {+ return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)+ }++ hashes := []string{}+ err = HashLister(ctx, ht, base64, download, f, stringWriter{&hashes})+ out = rc.Params{+ "hashType": ht.String(),+ "hashsum": hashes,+ }+ return out, err+}diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.goindex aec817b18..990cb9c6e 100644--- a/fs/operations/rc_test.go+++ b/fs/operations/rc_test.go@@ -2,6 +2,7 @@ package operations_testimport ("context"+ "fmt""net/http""net/http/httptest""net/url"@@ -14,6 +15,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/cache"+ "github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/rc""github.com/rclone/rclone/fstest"@@ -779,3 +781,88 @@ deadbeefcafe00000000000000000000 subdir/file2})}++// operations/hashsum: hashsum a directory+func TestRcHashsum(t *testing.T) {+ ctx := context.Background()+ r, call := rcNewRun(t, "operations/hashsum")+ r.Mkdir(ctx, r.Fremote)++ file1Contents := "file1 contents"+ file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1)+ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, file1)++ hasher := hash.NewMultiHasher()+ _, err := hasher.Write([]byte(file1Contents))+ require.NoError(t, err)++ for _, test := range []struct {+ ht hash.Type+ base64 bool+ download bool+ }{+ {+ ht: r.Fremote.Hashes().GetOne(),+ }, {+ ht: r.Fremote.Hashes().GetOne(),+ base64: true,+ }, {+ ht: hash.Whirlpool,+ base64: false,+ download: true,+ }, {+ ht: hash.Whirlpool,+ base64: true,+ download: true,+ },+ } {+ t.Run(fmt.Sprintf("hash=%v,base64=%v,download=%v", test.ht, test.base64, test.download), func(t *testing.T) {+ file1Hash, err := hasher.SumString(test.ht, test.base64)+ require.NoError(t, err)++ in := rc.Params{+ "fs": r.FremoteName,+ "hashType": test.ht.String(),+ "base64": test.base64,+ "download": test.download,+ }++ out, err := call.Fn(ctx, in)+ require.NoError(t, err)+ assert.Equal(t, test.ht.String(), out["hashType"])+ want := []string{+ fmt.Sprintf("%s hashsum-file1", file1Hash),+ }+ assert.Equal(t, want, out["hashsum"])+ })+ }+}++// operations/hashsum: hashsum a single file+func TestRcHashsumFile(t *testing.T) {+ ctx := context.Background()+ r, call := rcNewRun(t, "operations/hashsum")+ r.Mkdir(ctx, r.Fremote)++ file1Contents := "file1 contents"+ file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1)+ file2Contents := "file2 contents"+ file2 := r.WriteBoth(ctx, "hashsum-file2", file2Contents, t1)+ r.CheckLocalItems(t, file1, file2)+ r.CheckRemoteItems(t, file1, file2)++ // Make an fs pointing to just the file+ fsString := path.Join(r.FremoteName, file1.Path)++ in := rc.Params{+ "fs": fsString,+ "hashType": "MD5",+ "download": true,+ }++ out, err := call.Fn(ctx, in)+ require.NoError(t, err)+ assert.Equal(t, "md5", out["hashType"])+ assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595 hashsum-file1"}, out["hashsum"])+}