Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(gateway): hold the memory store's keys as hashes and evict by generation (#1270)

kyle-elliott-asymptotec03e650d4e67
diff --git a/gateway/deploy/beacon-gateway.yaml b/gateway/deploy/beacon-gateway.yaml
index 0a3f15a1..08f40089 100644
--- a/gateway/deploy/beacon-gateway.yaml
+++ b/gateway/deploy/beacon-gateway.yaml
@@ -42,7 +42,8 @@ pipeline:
seen_ttl: 6h
# All cross-call state (what a session already reported) lives here; replicas keep none of their
-# own. "memory" is per process.
+# own. "memory" is per process: about 50 bytes a key, so the default million keys is about 50 MB.
+# When full it forgets the keys least recently used; a forgotten key means content reported again.
store:
type: memory
max_entries: 1000000
diff --git a/gateway/store/memory.go b/gateway/store/memory.go
index 69d854b2..0982f63d 100644
--- a/gateway/store/memory.go
+++ b/gateway/store/memory.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"hash/maphash"
+ "math"
"sync"
"time"
)
@@ -12,50 +13,101 @@ const shards = 64
// Memory is a Store held in process memory: the default, for a single replica or for replicas
// that tolerate each other reporting the same content once more. It holds at most maxEntries
-// keys; past that it drops expired keys first, then arbitrary ones, so it behaves as a bounded
+// keys; past that it forgets the keys least recently written or read, so it behaves as a bounded
// cache rather than growing with the number of sessions it has ever seen.
+//
+// A gateway fills it with a key per session, per piece of content the session reported and per
+// turn, so it is sized for many small entries. A key is held as a 128-bit hash of it, which no
+// two of even billions of keys share by chance, and an entry as one word: its expiry, and its
+// value when that is at most a byte, as every mark the pipeline writes is. A longer value is kept
+// beside it. Holding the key strings and a copy of each value took about 210 bytes an entry, and
+// at the default million entries a fifth of a gigabyte that the collector traced on every cycle.
+//
+// Each shard keeps its keys in a ring of up to generations generations, each of an eighth of the
+// shard's keys. Keys are written to the newest, and a key read or added again moves to it; when
+// it is full a new one begins. The oldest is dropped whole: as soon as every key in it has
+// expired, and otherwise only when the shard is full, so a key is forgotten before it expires
+// only when the shard's live keys fill all but the one generation the expired ones may still
+// occupy (seven eighths of its keys). Keys are not deleted one at a time to make room, which left
+// a map full of tombstones and half-empty tables, and a full shard is never swept under its lock.
type Memory struct {
- seed maphash.Seed
- limit int // per shard
+ seeds [2]maphash.Seed
+ limit int // keys per shard
+ genCap int // keys per generation
now func() time.Time
shards [shards]shard
}
+// generations is how many generations a shard keeps at most. The expired keys of one generation
+// can hold room live keys need, so a shard keeps its live keys up to all but 1/generations of its
+// limit; looking a key up asks each generation it might be in.
+const generations = 8
+
type shard struct {
mu sync.Mutex
- m map[string]item
+ // ring holds the generations from ring[head] (the oldest) to ring[(head+count-1)%generations]
+ // (the newest).
+ ring [generations]generation
+ head, count int
+ n int // keys held
}
-type item struct {
- value []byte
- expires int64 // unix nanoseconds
+// generation is one generation of a shard's keys, with the values longer than a byte beside them,
+// and the latest expiry of any key written to it (Unix milliseconds).
+type generation struct {
+ m map[keyHash]entry
+ values map[keyHash][]byte
+ latest int64
}
-// NewMemory returns a Memory holding at most maxEntries keys.
+// keyHash identifies a key by two independent 64-bit hashes of it.
+type keyHash [2]uint64
+
+// entry packs an entry into a word: its expiry in Unix milliseconds, rounded up, above valueBits,
+// and below them how its value is held, and the value itself when it is a single byte.
+type entry uint64
+
+const (
+ valueBits = 10
+ // Value kinds, in bits 8 and 9.
+ valueNone = 0 << 8 // empty or nil: Get returns nil
+ valueByte = 1 << 8 // the low byte is the value
+ valueLong = 2 << 8 // the value is in generation.values
+ valueKinds = 3 << 8
+)
+
+func (e entry) expires() int64 { return int64(e >> valueBits) }
+func (e entry) kind() entry { return e & valueKinds }
+
+// NewMemory returns a Memory holding at most maxEntries keys, and never fewer than one a shard
+// (64).
func NewMemory(maxEntries int) *Memory {
- m := &Memory{seed: maphash.MakeSeed(), limit: max(1, maxEntries/shards), now: time.Now}
- for i := range m.shards {
- m.shards[i].m = make(map[string]item)
+ limit := max(1, maxEntries/shards)
+ return &Memory{
+ seeds: [2]maphash.Seed{maphash.MakeSeed(), maphash.MakeSeed()},
+ limit: limit,
+ genCap: max(1, limit/generations),
+ now: time.Now,
}
- return m
}
-func (m *Memory) shard(key string) *shard {
- return &m.shards[maphash.String(m.seed, key)%shards]
+func (m *Memory) hash(key string) (keyHash, *shard) {
+ h := keyHash{maphash.String(m.seeds[0], key), maphash.String(m.seeds[1], key)}
+ return h, &m.shards[h[0]%shards]
}
// Get implements Store.
func (m *Memory) Get(_ context.Context, keys ...string) ([][]byte, error) {
- now := m.now().UnixNano()
+ now := m.now().UnixMilli()
out := make([][]byte, len(keys))
for i, k := range keys {
- s := m.shard(k)
+ h, s := m.hash(k)
s.mu.Lock()
- if it, ok := s.m[k]; ok {
- if it.expires > now {
- out[i] = append([]byte(nil), it.value...)
+ if e, value, ok := m.live(s, h, now); ok && e.kind() != valueNone {
+ if e.kind() == valueByte {
+ out[i] = []byte{byte(e)}
} else {
- delete(s.m, k)
+ out[i] = append([]byte(nil), value...)
}
}
s.mu.Unlock()
@@ -65,14 +117,18 @@ func (m *Memory) Get(_ context.Context, keys ...string) ([][]byte, error) {
// Set implements Store.
func (m *Memory) Set(_ context.Context, entries ...Entry) error {
- now := m.now().UnixNano()
+ t := m.now()
+ now, nowMillis := t.UnixNano(), t.UnixMilli()
for _, e := range entries {
if e.TTL <= 0 {
return fmt.Errorf("store: key %q: TTL must be positive", e.Key)
}
- s := m.shard(e.Key)
+ h, s := m.hash(e.Key)
s.mu.Lock()
- s.put(e, now, m.limit)
+ if g, _, ok := s.find(h); ok && g != s.newest() {
+ s.delete(g, h)
+ }
+ m.put(s, h, pack(e, now), e.Value, nowMillis)
s.mu.Unlock()
}
return nil
@@ -80,16 +136,17 @@ func (m *Memory) Set(_ context.Context, entries ...Entry) error {
// Add implements Store.
func (m *Memory) Add(_ context.Context, entries ...Entry) ([]bool, error) {
- now := m.now().UnixNano()
+ t := m.now()
+ now, nowMillis := t.UnixNano(), t.UnixMilli()
added := make([]bool, len(entries))
for i, e := range entries {
if e.TTL <= 0 {
return nil, fmt.Errorf("store: key %q: TTL must be positive", e.Key)
}
- s := m.shard(e.Key)
+ h, s := m.hash(e.Key)
s.mu.Lock()
- if it, ok := s.m[e.Key]; !ok || it.expires <= now {
- s.put(e, now, m.limit)
+ if _, _, ok := m.live(s, h, nowMillis); !ok {
+ m.put(s, h, pack(e, now), e.Value, nowMillis)
added[i] = true
}
s.mu.Unlock()
@@ -97,21 +154,121 @@ func (m *Memory) Add(_ context.Context, entries ...Entry) ([]bool, error) {
return added, nil
}
-// put stores e, making room first when the shard is full: expired keys go, then arbitrary keys
-// down to 90% of the limit, so the cost of a full sweep is spread over many inserts.
-func (s *shard) put(e Entry, now int64, limit int) {
- if _, exists := s.m[e.Key]; !exists && len(s.m) >= limit {
- for k, it := range s.m {
- if it.expires <= now {
- delete(s.m, k)
- }
+// pack is e's entry, written at now (Unix nanoseconds). Its expiry is rounded up to the
+// millisecond, so a key never expires before it should, and outlives its TTL by less than one.
+func pack(e Entry, now int64) entry {
+ expires := now + int64(e.TTL)
+ if expires < now {
+ expires = math.MaxInt64
+ }
+ millis := expires / int64(time.Millisecond)
+ if expires%int64(time.Millisecond) != 0 {
+ millis++
+ }
+ packed := entry(millis) << valueBits
+ switch len(e.Value) {
+ case 0:
+ return packed | valueNone
+ case 1:
+ return packed | valueByte | entry(e.Value[0])
+ }
+ return packed | valueLong
+}
+
+// live finds h's entry, unexpired at now (Unix milliseconds), moving it to the newest generation
+// if it was in an older one; an expired entry is dropped. value is a long value as the shard
+// holds it.
+func (m *Memory) live(s *shard, h keyHash, now int64) (entry, []byte, bool) {
+ g, e, ok := s.find(h)
+ if !ok {
+ return 0, nil, false
+ }
+ value := g.values[h]
+ if e.expires() <= now {
+ s.delete(g, h)
+ return 0, nil, false
+ }
+ if g != s.newest() {
+ s.delete(g, h)
+ m.put(s, h, e, value, now)
+ }
+ return e, value, true
+}
+
+// put writes h's entry to the newest generation, which must not hold h in an older one. First it
+// drops the oldest generations whose keys have all expired, and makes room: a new generation when
+// the newest is full, and, when the shard is full, the oldest generation, live or not. value is
+// copied when the entry holds it beside itself.
+func (m *Memory) put(s *shard, h keyHash, e entry, value []byte, now int64) {
+ for s.count > 0 && (s.ring[s.head].latest <= now || len(s.ring[s.head].m) == 0) {
+ s.drop()
+ }
+ g := s.newest()
+ if g == nil || !g.has(h) {
+ for s.n >= m.limit {
+ s.drop()
}
- for k := range s.m {
- if len(s.m) < limit*9/10 {
- break
+ if g = s.newest(); g == nil || len(g.m) >= m.genCap {
+ if s.count == generations {
+ s.drop()
}
- delete(s.m, k)
+ g = &s.ring[(s.head+s.count)%generations]
+ *g = generation{m: make(map[keyHash]entry)}
+ s.count++
+ }
+ s.n++
+ }
+ if e.kind() == valueLong {
+ if g.values == nil {
+ g.values = make(map[keyHash][]byte)
+ }
+ g.values[h] = append([]byte(nil), value...)
+ } else {
+ delete(g.values, h)
+ }
+ g.m[h] = e
+ g.latest = max(g.latest, e.expires())
+}
+
+func (g *generation) has(h keyHash) bool {
+ _, ok := g.m[h]
+ return ok
+}
+
+// find returns the generation holding h, newest first, and h's entry there.
+func (s *shard) find(h keyHash) (*generation, entry, bool) {
+ for i := s.count - 1; i >= 0; i-- {
+ g := &s.ring[(s.head+i)%generations]
+ if e, ok := g.m[h]; ok {
+ return g, e, true
}
}
- s.m[e.Key] = item{value: append([]byte(nil), e.Value...), expires: now + int64(e.TTL)}
+ return nil, 0, false
}
+
+// newest is the newest generation, nil when there is none.
+func (s *shard) newest() *generation {
+ if s.count == 0 {
+ return nil
+ }
+ return &s.ring[(s.head+s.count-1)%generations]
+}
+
+// delete removes h from generation g.
+func (s *shard) delete(g *generation, h keyHash) {
+ delete(g.m, h)
+ delete(g.values, h)
+ s.n--
+}
+
+// drop drops the oldest generation whole.
+func (s *shard) drop() {
+ g := &s.ring[s.head]
+ s.n -= len(g.m)
+ *g = generation{}
+ s.head = (s.head + 1) % generations
+ s.count--
+}
+
+// len is how many keys the shard holds.
+func (s *shard) len() int { return s.n }
diff --git a/gateway/store/memory_test.go b/gateway/store/memory_test.go
index 534a5908..1edfde25 100644
--- a/gateway/store/memory_test.go
+++ b/gateway/store/memory_test.go
@@ -43,23 +43,124 @@ func TestExpiry(t *testing.T) {
}
}
-// The store is bounded: writing far more keys than the limit keeps it near the limit, and the
-// newest key written is always readable.
+// The store is bounded: writing far more keys than the limit keeps it within the limit (and its
+// floor of one key a shard), and the newest key written is always readable.
func TestBounded(t *testing.T) {
- m := NewMemory(shards * 100)
ctx := context.Background()
- for i := range 50_000 {
- key := fmt.Sprint("k", i)
- _ = m.Set(ctx, Entry{key, []byte("v"), time.Hour})
- if vals, _ := m.Get(ctx, key); vals[0] == nil {
- t.Fatalf("key %s missing right after Set", key)
+ for _, limit := range []int{shards * 100, 1000, 127, 100, 64, 10} {
+ m := NewMemory(limit)
+ for i := range 20 * max(limit, shards) {
+ key := fmt.Sprint("k", i)
+ _ = m.Set(ctx, Entry{key, []byte("v"), time.Hour})
+ if vals, _ := m.Get(ctx, key); vals[0] == nil {
+ t.Fatalf("limit %d: key %s missing right after Set", limit, key)
+ }
+ }
+ total := 0
+ for i := range m.shards {
+ total += m.shards[i].len()
+ }
+ if bound := max(limit, shards); total > bound {
+ t.Errorf("limit %d: store holds %d keys, more than %d", limit, total, bound)
}
}
- total := 0
+}
+
+// A full store forgets the keys least recently used: a key read between writes of many times the
+// limit is kept, and a key written once and never read again is not.
+func TestKeepsKeysInUse(t *testing.T) {
+ m := NewMemory(shards * 10)
+ ctx := context.Background()
+ _ = m.Set(ctx, Entry{"hot", []byte("v"), time.Hour}, Entry{"cold", []byte("v"), time.Hour})
+ for i := range shards * 100 {
+ _ = m.Set(ctx, Entry{fmt.Sprint("k", i), []byte("v"), time.Hour})
+ if vals, _ := m.Get(ctx, "hot"); vals[0] == nil {
+ t.Fatalf("a key read after every write was evicted after %d writes", i+1)
+ }
+ }
+ if vals, _ := m.Get(ctx, "cold"); vals[0] != nil {
+ t.Error("a key never read again outlived a hundred times the limit in writes")
+ }
+}
+
+// Values of every length read back as written, as copies, and replacing a long value with a short
+// one leaves nothing of it behind.
+func TestValues(t *testing.T) {
+ m := NewMemory(1000)
+ ctx := context.Background()
+ long := []byte("a value longer than a byte")
+ _ = m.Set(ctx, Entry{"empty", nil, time.Hour}, Entry{"byte", []byte{0xFF}, time.Hour}, Entry{"long", long, time.Hour})
+ long[0] = 'X' // the store copied it
+ vals, _ := m.Get(ctx, "empty", "byte", "long")
+ if vals[0] != nil || !slices.Equal(vals[1], []byte{0xFF}) || string(vals[2]) != "a value longer than a byte" {
+ t.Fatalf("Get = %q", vals)
+ }
+ vals[2][0] = 'Y' // and Get returned a copy
+ if again, _ := m.Get(ctx, "long"); string(again[0]) != "a value longer than a byte" {
+ t.Errorf("value changed through Get's result: %q", again[0])
+ }
+ _ = m.Set(ctx, Entry{"long", []byte{1}, time.Hour})
+ if vals, _ := m.Get(ctx, "long"); !slices.Equal(vals[0], []byte{1}) {
+ t.Errorf("replaced value = %q", vals[0])
+ }
+ held := 0
for i := range m.shards {
- total += len(m.shards[i].m)
+ for _, g := range m.shards[i].ring {
+ held += len(g.values)
+ }
+ }
+ if held != 0 {
+ t.Errorf("%d long values held after the only one was replaced", held)
}
- if total > shards*100 {
- t.Errorf("store holds %d keys, limit %d", total, shards*100)
+}
+
+// A key is forgotten before it expires only when live keys fill the store: in a steady state of
+// keys written at a constant rate and expiring a TTL later, with the live ones at 70% of the limit,
+// every key written within the last TTL and never read since is still held.
+func TestKeepsLiveKeysUntilFull(t *testing.T) {
+ const (
+ limit = shards * 2000
+ ttl = 1000 // ms
+ live = limit * 7 / 10
+ )
+ m := NewMemory(limit)
+ start := time.Unix(1000, 0)
+ now := start
+ m.now = func() time.Time { return now }
+ ctx := context.Background()
+ written := 0
+ for ms := range 4 * ttl {
+ now = start.Add(time.Duration(ms) * time.Millisecond)
+ for ; written < (ms+1)*live/ttl; written++ {
+ _, _ = m.Add(ctx, Entry{fmt.Sprint("k", written), []byte{1}, ttl * time.Millisecond})
+ }
+ }
+ lost, from := 0, written-live+live/ttl // the keys written within the last TTL
+ for i := from; i < written; i++ {
+ if vals, _ := m.Get(ctx, fmt.Sprint("k", i)); vals[0] == nil {
+ lost++
+ }
+ }
+ if lost > 0 {
+ t.Errorf("%d of %d live keys forgotten with live keys at 70%% of the limit", lost, written-from)
+ }
+}
+
+// A key's expiry is kept to the millisecond, rounded up, so a key outlives its TTL by less than a
+// millisecond and never expires before it, wherever in a millisecond it was written.
+func TestShortTTL(t *testing.T) {
+ m := NewMemory(1000)
+ written := time.Unix(1000, 900_000) // 0.9 ms into a millisecond
+ now := written
+ m.now = func() time.Time { return now }
+ ctx := context.Background()
+ _ = m.Set(ctx, Entry{"k", []byte("v"), time.Millisecond}) // expires 1.9 ms into it
+ now = written.Add(950 * time.Microsecond)
+ if vals, _ := m.Get(ctx, "k"); vals[0] == nil {
+ t.Error("key expired before its TTL")
+ }
+ now = written.Add(2 * time.Millisecond)
+ if vals, _ := m.Get(ctx, "k"); vals[0] != nil {
+ t.Error("key outlived its TTL by a millisecond or more")
}
}