SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
perf(weed/storage/needle): intern the stored ttl values (#10611)
Chris Lu33c36fc7a3a3
diff --git a/weed/storage/needle/volume_ttl.go b/weed/storage/needle/volume_ttl.goindex 104efd2c46..600dd38a69 100644--- a/weed/storage/needle/volume_ttl.go+++ b/weed/storage/needle/volume_ttl.go@@ -88,20 +88,39 @@ func fitTtlCount(count int, unit byte) *TTL {return EMPTY_TTL}-// read stored bytes to a ttl-func LoadTTLFromBytes(input []byte) (t *TTL) {- if input[0] == 0 && input[1] == 0 {+// storedTTLs interns every count paired with a defined unit. The master decodes+// one per volume in every heartbeat and keeps it for the volume's lifetime, so+// sharing spares a cluster with millions of volume replicas a distinct two-byte+// object for each.+//+// Entries are immutable. Callers must not write through the returned pointer.+var storedTTLs = func() (table [256][Year + 1]TTL) {+ for count := range table {+ for unit := range table[count] {+ table[count][unit] = TTL{Count: byte(count), Unit: byte(unit)}+ }+ }+ return table+}()++func loadTTL(count, unit byte) *TTL {+ if count == 0 && unit == 0 {return EMPTY_TTL}- return &TTL{Count: input[0], Unit: input[1]}+ if unit <= Year {+ return &storedTTLs[count][unit]+ }+ return &TTL{Count: count, Unit: unit}+}++// read stored bytes to a ttl+func LoadTTLFromBytes(input []byte) (t *TTL) {+ return loadTTL(input[0], input[1])}// read stored bytes to a ttlfunc LoadTTLFromUint32(ttl uint32) (t *TTL) {- input := make([]byte, 2)- input[1] = byte(ttl)- input[0] = byte(ttl >> 8)- return LoadTTLFromBytes(input)+ return loadTTL(byte(ttl>>8), byte(ttl))}// save stored bytes to an output with 2 bytesdiff --git a/weed/storage/needle/volume_ttl_test.go b/weed/storage/needle/volume_ttl_test.goindex bfa97924c0..45f1606ee5 100644--- a/weed/storage/needle/volume_ttl_test.go+++ b/weed/storage/needle/volume_ttl_test.go@@ -73,3 +73,26 @@ func TestTTLReadWrite(t *testing.T) {}}++func TestLoadTTLIsInterned(t *testing.T) {+ for count := 0; count < 256; count++ {+ for unit := 0; unit < 256; unit++ {+ got := LoadTTLFromBytes([]byte{byte(count), byte(unit)})+ if got.Count != byte(count) || got.Unit != byte(unit) {+ if count == 0 && unit == 0 {+ continue+ }+ t.Fatalf("count %d unit %d: got %+v", count, unit, got)+ }+ if fromUint32 := LoadTTLFromUint32(got.ToUint32()); count != 0 && *fromUint32 != *got {+ t.Fatalf("count %d unit %d: uint32 round trip gave %+v", count, unit, fromUint32)+ }+ }+ }+ if LoadTTLFromBytes([]byte{3, Day}) != LoadTTLFromBytes([]byte{3, Day}) {+ t.Error("expected repeated loads of the same stored ttl to share one value")+ }+ if LoadTTLFromBytes([]byte{0, 0}) != EMPTY_TTL {+ t.Error("expected a zero ttl to stay EMPTY_TTL")+ }+}