"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
hash: add support for BLAKE3, XXH3, XXH128
albertonydf9c921dd5b4
diff --git a/fs/hash/hash.go b/fs/hash/hash.goindex 82caafd2c..2f88cb0d6 100644--- a/fs/hash/hash.go+++ b/fs/hash/hash.go@@ -16,6 +16,8 @@ import ("strings""github.com/jzelinskie/whirlpool"+ "github.com/zeebo/blake3"+ "github.com/zeebo/xxh3")// Type indicates a standard hashing algorithm@@ -91,8 +93,27 @@ var (// SHA512 indicates SHA-512 supportSHA512 Type++ // BLAKE3 indicates BLAKE3 support+ BLAKE3 Type++ // XXH3 indicates XXH3 support, also known as XXH3-64, a variant of xxHash+ XXH3 Type++ // XXH128 indicates XXH128 support, also known as XXH3-128, a variant of xxHash+ XXH128 Type)+type xxh128Hasher struct {+ xxh3.Hasher+}++// Sum overrides xxh3.Sum to return value based on Sum128 instead of the default Sum64.+func (h *xxh128Hasher) Sum(b []byte) []byte {+ buf := h.Sum128().Bytes()+ return buf[:]+}+func init() {MD5 = RegisterHash("md5", "MD5", 32, md5.New)SHA1 = RegisterHash("sha1", "SHA-1", 40, sha1.New)@@ -100,6 +121,9 @@ func init() {CRC32 = RegisterHash("crc32", "CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() })SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New)SHA512 = RegisterHash("sha512", "SHA-512", 128, sha512.New)+ BLAKE3 = RegisterHash("blake3", "BLAKE3", 64, func() hash.Hash { return blake3.New() })+ XXH3 = RegisterHash("xxh3", "XXH3", 16, func() hash.Hash { return xxh3.New() })+ XXH128 = RegisterHash("xxh128", "XXH128", 32, func() hash.Hash { return &xxh128Hasher{} })}// Supported returns a set of all the supported hashes bydiff --git a/fs/hash/hash_test.go b/fs/hash/hash_test.goindex 8fbd71bf9..a5e5f8cb7 100644--- a/fs/hash/hash_test.go+++ b/fs/hash/hash_test.go@@ -78,6 +78,9 @@ var hashTestSet = []hashTest{hash.CRC32: "a6041d7e",hash.SHA256: "c839e57675862af5c21bd0a15413c3ec579e0d5522dab600bc6c3489b05b8f54",hash.SHA512: "008e7e9b5d94d37bf5e07c955890f730f137a41b8b0db16cb535a9b4cb5632c2bccff31685ec470130fe10e2258a0ab50ab587472258f3132ccf7d7d59fb91db",+ hash.BLAKE3: "0a7276a407a3be1b4d31488318ee05a335aad5a3b82c4420e592a8178c9e86bb",+ hash.XXH3: "4b83b0c51c543525",+ hash.XXH128: "438de241a57d684214f67657f7aad93b",},},// Empty data set@@ -90,6 +93,9 @@ var hashTestSet = []hashTest{hash.CRC32: "00000000",hash.SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",hash.SHA512: "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",+ hash.BLAKE3: "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",+ hash.XXH3: "2d06800538d394c2",+ hash.XXH128: "99aa06d3014798d86001c324468d497f",},},}diff --git a/go.mod b/go.modindex 4bcd5ec62..d474992a7 100644--- a/go.mod+++ b/go.mod@@ -78,6 +78,7 @@ require (github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78github.com/yunify/qingstor-sdk-go/v3 v3.2.0github.com/zeebo/blake3 v0.2.4+ github.com/zeebo/xxh3 v1.0.2go.etcd.io/bbolt v1.4.0goftp.io/server/v2 v2.0.1golang.org/x/crypto v0.39.0diff --git a/go.sum b/go.sumindex b0af1475d..ee6695ae2 100644--- a/go.sum+++ b/go.sum@@ -650,6 +650,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw=