SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
storage: hash index-metric bloom filter with xxhash
Chris Luab559bca76d5
diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.goindex ed744162d5..498ae49cc2 100644--- a/weed/storage/needle_map_metric.go+++ b/weed/storage/needle_map_metric.go@@ -6,6 +6,7 @@ import ("os""sync/atomic"+ "github.com/cespare/xxhash/v2""github.com/seaweedfs/seaweedfs/weed/storage/idx""github.com/seaweedfs/seaweedfs/weed/storage/needle". "github.com/seaweedfs/seaweedfs/weed/storage/types"@@ -155,6 +156,12 @@ func needleMapMetricFromIndexFile(r *os.File, mm *mapMetric, version needle.Versvar liveKeyBytes uint64err := reverseWalkIndexFile(r, func(entryCount int64) {bf = boom.NewBloomFilter(uint(entryCount), 0.001)+ // The default FNV-1 kernel leaves the last byte of a big-endian+ // NeedleId as the final XOR, so keys in the same 256-block get+ // correlated probe positions and false positives arrive in bursts+ // of hundreds. xxhash diffuses the input fully and keeps the+ // observed rate near the 0.001 target.+ bf.SetHash(xxhash.New())}, func(key NeedleId, offset Offset, size Size) error {mm.MaybeSetMaxFileKey(key)