Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: add operations/hashsum to the rc as rclone hashsum equivalent

Nick Craig-Woodd50572b1083b
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
index 533c3601e..1435e66de 100644
--- a/fs/operations/rc.go
+++ b/fs/operations/rc.go
@@ -876,3 +876,80 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
}
return out, nil
}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/hashsum",
+ AuthRequired: true,
+ Fn: rcHashsum,
+ Title: "Produces a hashsum file for all the objects in the path.",
+ Help: `Produces a hash file for all the objects in the path using the hash
+named. The output is in the same format as the standard
+md5sum/sha1sum tool.
+
+This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
+ - this can point to a file and just that file will be returned in the listing.
+- hashType - type of hash to be used
+- download - check by downloading rather than with hash (boolean)
+- base64 - output the hashes in base64 rather than hex (boolean)
+
+If you supply the download flag, it will download the data from the
+remote and create the hash on the fly. This can be useful for remotes
+that don't support the given hash or if you really want to check all
+the data.
+
+Note that if you wish to supply a checkfile to check hashes against
+the current files then you should use operations/check instead of
+operations/hashsum.
+
+Returns:
+
+- hashsum - array of strings of the hashes
+- hashType - type of hash used
+
+Example:
+
+ $ rclone rc --loopback operations/hashsum fs=bin hashType=MD5 download=true base64=true
+ {
+ "hashType": "md5",
+ "hashsum": [
+ "WTSVLpuiXyJO_kGzJerRLg== backend-versions.sh",
+ "v1b_OlWCJO9LtNq3EIKkNQ== bisect-go-rclone.sh",
+ "VHbmHzHh4taXzgag8BAIKQ== bisect-rclone.sh",
+ ]
+ }
+
+See the [hashsum](/commands/rclone_hashsum/) command for more information on the above.
+`,
+ })
+}
+
+// Hashsum a directory
+func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs")
+ if err != nil {
+ return nil, err
+ }
+
+ download, _ := in.GetBool("download")
+ base64, _ := in.GetBool("base64")
+ hashType, err := in.GetString("hashType")
+ if err != nil {
+ return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)
+ }
+ var ht hash.Type
+ err = ht.Set(hashType)
+ if err != nil {
+ return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)
+ }
+
+ hashes := []string{}
+ err = HashLister(ctx, ht, base64, download, f, stringWriter{&hashes})
+ out = rc.Params{
+ "hashType": ht.String(),
+ "hashsum": hashes,
+ }
+ return out, err
+}
diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go
index aec817b18..990cb9c6e 100644
--- a/fs/operations/rc_test.go
+++ b/fs/operations/rc_test.go
@@ -2,6 +2,7 @@ package operations_test
import (
"context"
+ "fmt"
"net/http"
"net/http/httptest"
"net/url"
@@ -14,6 +15,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fstest"
@@ -779,3 +781,88 @@ deadbeefcafe00000000000000000000 subdir/file2
})
}
+
+// operations/hashsum: hashsum a directory
+func TestRcHashsum(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/hashsum")
+ r.Mkdir(ctx, r.Fremote)
+
+ file1Contents := "file1 contents"
+ file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file1)
+
+ hasher := hash.NewMultiHasher()
+ _, err := hasher.Write([]byte(file1Contents))
+ require.NoError(t, err)
+
+ for _, test := range []struct {
+ ht hash.Type
+ base64 bool
+ download bool
+ }{
+ {
+ ht: r.Fremote.Hashes().GetOne(),
+ }, {
+ ht: r.Fremote.Hashes().GetOne(),
+ base64: true,
+ }, {
+ ht: hash.Whirlpool,
+ base64: false,
+ download: true,
+ }, {
+ ht: hash.Whirlpool,
+ base64: true,
+ download: true,
+ },
+ } {
+ t.Run(fmt.Sprintf("hash=%v,base64=%v,download=%v", test.ht, test.base64, test.download), func(t *testing.T) {
+ file1Hash, err := hasher.SumString(test.ht, test.base64)
+ require.NoError(t, err)
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "hashType": test.ht.String(),
+ "base64": test.base64,
+ "download": test.download,
+ }
+
+ out, err := call.Fn(ctx, in)
+ require.NoError(t, err)
+ assert.Equal(t, test.ht.String(), out["hashType"])
+ want := []string{
+ fmt.Sprintf("%s hashsum-file1", file1Hash),
+ }
+ assert.Equal(t, want, out["hashsum"])
+ })
+ }
+}
+
+// operations/hashsum: hashsum a single file
+func TestRcHashsumFile(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/hashsum")
+ r.Mkdir(ctx, r.Fremote)
+
+ file1Contents := "file1 contents"
+ file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1)
+ file2Contents := "file2 contents"
+ file2 := r.WriteBoth(ctx, "hashsum-file2", file2Contents, t1)
+ r.CheckLocalItems(t, file1, file2)
+ r.CheckRemoteItems(t, file1, file2)
+
+ // Make an fs pointing to just the file
+ fsString := path.Join(r.FremoteName, file1.Path)
+
+ in := rc.Params{
+ "fs": fsString,
+ "hashType": "MD5",
+ "download": true,
+ }
+
+ out, err := call.Fn(ctx, in)
+ require.NoError(t, err)
+ assert.Equal(t, "md5", out["hashType"])
+ assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595 hashsum-file1"}, out["hashsum"])
+}