Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

storage: hash index-metric bloom filter with xxhash

Chris Luab559bca76d5
diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.go
index ed744162d5..498ae49cc2 100644
--- a/weed/storage/needle_map_metric.go
+++ b/weed/storage/needle_map_metric.go
@@ -6,6 +6,7 @@ import (
"os"
"sync/atomic"
+ "github.com/cespare/xxhash/v2"
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
. "github.com/seaweedfs/seaweedfs/weed/storage/types"
@@ -155,6 +156,12 @@ func needleMapMetricFromIndexFile(r *os.File, mm *mapMetric, version needle.Vers
var liveKeyBytes uint64
err := reverseWalkIndexFile(r, func(entryCount int64) {
bf = boom.NewBloomFilter(uint(entryCount), 0.001)
+ // The default FNV-1 kernel leaves the last byte of a big-endian
+ // NeedleId as the final XOR, so keys in the same 256-block get
+ // correlated probe positions and false positives arrive in bursts
+ // of hundreds. xxhash diffuses the input fully and keeps the
+ // observed rate near the 0.001 target.
+ bf.SetHash(xxhash.New())
}, func(key NeedleId, offset Offset, size Size) error {
mm.MaybeSetMaxFileKey(key)