The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
perf(gateway): hold the memory store's keys as hashes and evict by generation (#1270)
kyle-elliott-asymptotec03e650d4e67
diff --git a/gateway/deploy/beacon-gateway.yaml b/gateway/deploy/beacon-gateway.yamlindex 0a3f15a1..08f40089 100644--- a/gateway/deploy/beacon-gateway.yaml+++ b/gateway/deploy/beacon-gateway.yaml@@ -42,7 +42,8 @@ pipeline:seen_ttl: 6h# All cross-call state (what a session already reported) lives here; replicas keep none of their-# own. "memory" is per process.+# own. "memory" is per process: about 50 bytes a key, so the default million keys is about 50 MB.+# When full it forgets the keys least recently used; a forgotten key means content reported again.store:type: memorymax_entries: 1000000diff --git a/gateway/store/memory.go b/gateway/store/memory.goindex 69d854b2..0982f63d 100644--- a/gateway/store/memory.go+++ b/gateway/store/memory.go@@ -4,6 +4,7 @@ import ("context""fmt""hash/maphash"+ "math""sync""time")@@ -12,50 +13,101 @@ const shards = 64// Memory is a Store held in process memory: the default, for a single replica or for replicas// that tolerate each other reporting the same content once more. It holds at most maxEntries-// keys; past that it drops expired keys first, then arbitrary ones, so it behaves as a bounded+// keys; past that it forgets the keys least recently written or read, so it behaves as a bounded// cache rather than growing with the number of sessions it has ever seen.+//+// A gateway fills it with a key per session, per piece of content the session reported and per+// turn, so it is sized for many small entries. A key is held as a 128-bit hash of it, which no+// two of even billions of keys share by chance, and an entry as one word: its expiry, and its+// value when that is at most a byte, as every mark the pipeline writes is. A longer value is kept+// beside it. Holding the key strings and a copy of each value took about 210 bytes an entry, and+// at the default million entries a fifth of a gigabyte that the collector traced on every cycle.+//+// Each shard keeps its keys in a ring of up to generations generations, each of an eighth of the+// shard's keys. Keys are written to the newest, and a key read or added again moves to it; when+// it is full a new one begins. The oldest is dropped whole: as soon as every key in it has+// expired, and otherwise only when the shard is full, so a key is forgotten before it expires+// only when the shard's live keys fill all but the one generation the expired ones may still+// occupy (seven eighths of its keys). Keys are not deleted one at a time to make room, which left+// a map full of tombstones and half-empty tables, and a full shard is never swept under its lock.type Memory struct {- seed maphash.Seed- limit int // per shard+ seeds [2]maphash.Seed+ limit int // keys per shard+ genCap int // keys per generationnow func() time.Timeshards [shards]shard}+// generations is how many generations a shard keeps at most. The expired keys of one generation+// can hold room live keys need, so a shard keeps its live keys up to all but 1/generations of its+// limit; looking a key up asks each generation it might be in.+const generations = 8+type shard struct {mu sync.Mutex- m map[string]item+ // ring holds the generations from ring[head] (the oldest) to ring[(head+count-1)%generations]+ // (the newest).+ ring [generations]generation+ head, count int+ n int // keys held}-type item struct {- value []byte- expires int64 // unix nanoseconds+// generation is one generation of a shard's keys, with the values longer than a byte beside them,+// and the latest expiry of any key written to it (Unix milliseconds).+type generation struct {+ m map[keyHash]entry+ values map[keyHash][]byte+ latest int64}-// NewMemory returns a Memory holding at most maxEntries keys.+// keyHash identifies a key by two independent 64-bit hashes of it.+type keyHash [2]uint64++// entry packs an entry into a word: its expiry in Unix milliseconds, rounded up, above valueBits,+// and below them how its value is held, and the value itself when it is a single byte.+type entry uint64++const (+ valueBits = 10+ // Value kinds, in bits 8 and 9.+ valueNone = 0 << 8 // empty or nil: Get returns nil+ valueByte = 1 << 8 // the low byte is the value+ valueLong = 2 << 8 // the value is in generation.values+ valueKinds = 3 << 8+)++func (e entry) expires() int64 { return int64(e >> valueBits) }+func (e entry) kind() entry { return e & valueKinds }++// NewMemory returns a Memory holding at most maxEntries keys, and never fewer than one a shard+// (64).func NewMemory(maxEntries int) *Memory {- m := &Memory{seed: maphash.MakeSeed(), limit: max(1, maxEntries/shards), now: time.Now}- for i := range m.shards {- m.shards[i].m = make(map[string]item)+ limit := max(1, maxEntries/shards)+ return &Memory{+ seeds: [2]maphash.Seed{maphash.MakeSeed(), maphash.MakeSeed()},+ limit: limit,+ genCap: max(1, limit/generations),+ now: time.Now,}- return m}-func (m *Memory) shard(key string) *shard {- return &m.shards[maphash.String(m.seed, key)%shards]+func (m *Memory) hash(key string) (keyHash, *shard) {+ h := keyHash{maphash.String(m.seeds[0], key), maphash.String(m.seeds[1], key)}+ return h, &m.shards[h[0]%shards]}// Get implements Store.func (m *Memory) Get(_ context.Context, keys ...string) ([][]byte, error) {- now := m.now().UnixNano()+ now := m.now().UnixMilli()out := make([][]byte, len(keys))for i, k := range keys {- s := m.shard(k)+ h, s := m.hash(k)s.mu.Lock()- if it, ok := s.m[k]; ok {- if it.expires > now {- out[i] = append([]byte(nil), it.value...)+ if e, value, ok := m.live(s, h, now); ok && e.kind() != valueNone {+ if e.kind() == valueByte {+ out[i] = []byte{byte(e)}} else {- delete(s.m, k)+ out[i] = append([]byte(nil), value...)}}s.mu.Unlock()@@ -65,14 +117,18 @@ func (m *Memory) Get(_ context.Context, keys ...string) ([][]byte, error) {// Set implements Store.func (m *Memory) Set(_ context.Context, entries ...Entry) error {- now := m.now().UnixNano()+ t := m.now()+ now, nowMillis := t.UnixNano(), t.UnixMilli()for _, e := range entries {if e.TTL <= 0 {return fmt.Errorf("store: key %q: TTL must be positive", e.Key)}- s := m.shard(e.Key)+ h, s := m.hash(e.Key)s.mu.Lock()- s.put(e, now, m.limit)+ if g, _, ok := s.find(h); ok && g != s.newest() {+ s.delete(g, h)+ }+ m.put(s, h, pack(e, now), e.Value, nowMillis)s.mu.Unlock()}return nil@@ -80,16 +136,17 @@ func (m *Memory) Set(_ context.Context, entries ...Entry) error {// Add implements Store.func (m *Memory) Add(_ context.Context, entries ...Entry) ([]bool, error) {- now := m.now().UnixNano()+ t := m.now()+ now, nowMillis := t.UnixNano(), t.UnixMilli()added := make([]bool, len(entries))for i, e := range entries {if e.TTL <= 0 {return nil, fmt.Errorf("store: key %q: TTL must be positive", e.Key)}- s := m.shard(e.Key)+ h, s := m.hash(e.Key)s.mu.Lock()- if it, ok := s.m[e.Key]; !ok || it.expires <= now {- s.put(e, now, m.limit)+ if _, _, ok := m.live(s, h, nowMillis); !ok {+ m.put(s, h, pack(e, now), e.Value, nowMillis)added[i] = true}s.mu.Unlock()@@ -97,21 +154,121 @@ func (m *Memory) Add(_ context.Context, entries ...Entry) ([]bool, error) {return added, nil}-// put stores e, making room first when the shard is full: expired keys go, then arbitrary keys-// down to 90% of the limit, so the cost of a full sweep is spread over many inserts.-func (s *shard) put(e Entry, now int64, limit int) {- if _, exists := s.m[e.Key]; !exists && len(s.m) >= limit {- for k, it := range s.m {- if it.expires <= now {- delete(s.m, k)- }+// pack is e's entry, written at now (Unix nanoseconds). Its expiry is rounded up to the+// millisecond, so a key never expires before it should, and outlives its TTL by less than one.+func pack(e Entry, now int64) entry {+ expires := now + int64(e.TTL)+ if expires < now {+ expires = math.MaxInt64+ }+ millis := expires / int64(time.Millisecond)+ if expires%int64(time.Millisecond) != 0 {+ millis+++ }+ packed := entry(millis) << valueBits+ switch len(e.Value) {+ case 0:+ return packed | valueNone+ case 1:+ return packed | valueByte | entry(e.Value[0])+ }+ return packed | valueLong+}++// live finds h's entry, unexpired at now (Unix milliseconds), moving it to the newest generation+// if it was in an older one; an expired entry is dropped. value is a long value as the shard+// holds it.+func (m *Memory) live(s *shard, h keyHash, now int64) (entry, []byte, bool) {+ g, e, ok := s.find(h)+ if !ok {+ return 0, nil, false+ }+ value := g.values[h]+ if e.expires() <= now {+ s.delete(g, h)+ return 0, nil, false+ }+ if g != s.newest() {+ s.delete(g, h)+ m.put(s, h, e, value, now)+ }+ return e, value, true+}++// put writes h's entry to the newest generation, which must not hold h in an older one. First it+// drops the oldest generations whose keys have all expired, and makes room: a new generation when+// the newest is full, and, when the shard is full, the oldest generation, live or not. value is+// copied when the entry holds it beside itself.+func (m *Memory) put(s *shard, h keyHash, e entry, value []byte, now int64) {+ for s.count > 0 && (s.ring[s.head].latest <= now || len(s.ring[s.head].m) == 0) {+ s.drop()+ }+ g := s.newest()+ if g == nil || !g.has(h) {+ for s.n >= m.limit {+ s.drop()}- for k := range s.m {- if len(s.m) < limit*9/10 {- break+ if g = s.newest(); g == nil || len(g.m) >= m.genCap {+ if s.count == generations {+ s.drop()}- delete(s.m, k)+ g = &s.ring[(s.head+s.count)%generations]+ *g = generation{m: make(map[keyHash]entry)}+ s.count+++ }+ s.n+++ }+ if e.kind() == valueLong {+ if g.values == nil {+ g.values = make(map[keyHash][]byte)+ }+ g.values[h] = append([]byte(nil), value...)+ } else {+ delete(g.values, h)+ }+ g.m[h] = e+ g.latest = max(g.latest, e.expires())+}++func (g *generation) has(h keyHash) bool {+ _, ok := g.m[h]+ return ok+}++// find returns the generation holding h, newest first, and h's entry there.+func (s *shard) find(h keyHash) (*generation, entry, bool) {+ for i := s.count - 1; i >= 0; i-- {+ g := &s.ring[(s.head+i)%generations]+ if e, ok := g.m[h]; ok {+ return g, e, true}}- s.m[e.Key] = item{value: append([]byte(nil), e.Value...), expires: now + int64(e.TTL)}+ return nil, 0, false}++// newest is the newest generation, nil when there is none.+func (s *shard) newest() *generation {+ if s.count == 0 {+ return nil+ }+ return &s.ring[(s.head+s.count-1)%generations]+}++// delete removes h from generation g.+func (s *shard) delete(g *generation, h keyHash) {+ delete(g.m, h)+ delete(g.values, h)+ s.n--+}++// drop drops the oldest generation whole.+func (s *shard) drop() {+ g := &s.ring[s.head]+ s.n -= len(g.m)+ *g = generation{}+ s.head = (s.head + 1) % generations+ s.count--+}++// len is how many keys the shard holds.+func (s *shard) len() int { return s.n }diff --git a/gateway/store/memory_test.go b/gateway/store/memory_test.goindex 534a5908..1edfde25 100644--- a/gateway/store/memory_test.go+++ b/gateway/store/memory_test.go@@ -43,23 +43,124 @@ func TestExpiry(t *testing.T) {}}-// The store is bounded: writing far more keys than the limit keeps it near the limit, and the-// newest key written is always readable.+// The store is bounded: writing far more keys than the limit keeps it within the limit (and its+// floor of one key a shard), and the newest key written is always readable.func TestBounded(t *testing.T) {- m := NewMemory(shards * 100)ctx := context.Background()- for i := range 50_000 {- key := fmt.Sprint("k", i)- _ = m.Set(ctx, Entry{key, []byte("v"), time.Hour})- if vals, _ := m.Get(ctx, key); vals[0] == nil {- t.Fatalf("key %s missing right after Set", key)+ for _, limit := range []int{shards * 100, 1000, 127, 100, 64, 10} {+ m := NewMemory(limit)+ for i := range 20 * max(limit, shards) {+ key := fmt.Sprint("k", i)+ _ = m.Set(ctx, Entry{key, []byte("v"), time.Hour})+ if vals, _ := m.Get(ctx, key); vals[0] == nil {+ t.Fatalf("limit %d: key %s missing right after Set", limit, key)+ }+ }+ total := 0+ for i := range m.shards {+ total += m.shards[i].len()+ }+ if bound := max(limit, shards); total > bound {+ t.Errorf("limit %d: store holds %d keys, more than %d", limit, total, bound)}}- total := 0+}++// A full store forgets the keys least recently used: a key read between writes of many times the+// limit is kept, and a key written once and never read again is not.+func TestKeepsKeysInUse(t *testing.T) {+ m := NewMemory(shards * 10)+ ctx := context.Background()+ _ = m.Set(ctx, Entry{"hot", []byte("v"), time.Hour}, Entry{"cold", []byte("v"), time.Hour})+ for i := range shards * 100 {+ _ = m.Set(ctx, Entry{fmt.Sprint("k", i), []byte("v"), time.Hour})+ if vals, _ := m.Get(ctx, "hot"); vals[0] == nil {+ t.Fatalf("a key read after every write was evicted after %d writes", i+1)+ }+ }+ if vals, _ := m.Get(ctx, "cold"); vals[0] != nil {+ t.Error("a key never read again outlived a hundred times the limit in writes")+ }+}++// Values of every length read back as written, as copies, and replacing a long value with a short+// one leaves nothing of it behind.+func TestValues(t *testing.T) {+ m := NewMemory(1000)+ ctx := context.Background()+ long := []byte("a value longer than a byte")+ _ = m.Set(ctx, Entry{"empty", nil, time.Hour}, Entry{"byte", []byte{0xFF}, time.Hour}, Entry{"long", long, time.Hour})+ long[0] = 'X' // the store copied it+ vals, _ := m.Get(ctx, "empty", "byte", "long")+ if vals[0] != nil || !slices.Equal(vals[1], []byte{0xFF}) || string(vals[2]) != "a value longer than a byte" {+ t.Fatalf("Get = %q", vals)+ }+ vals[2][0] = 'Y' // and Get returned a copy+ if again, _ := m.Get(ctx, "long"); string(again[0]) != "a value longer than a byte" {+ t.Errorf("value changed through Get's result: %q", again[0])+ }+ _ = m.Set(ctx, Entry{"long", []byte{1}, time.Hour})+ if vals, _ := m.Get(ctx, "long"); !slices.Equal(vals[0], []byte{1}) {+ t.Errorf("replaced value = %q", vals[0])+ }+ held := 0for i := range m.shards {- total += len(m.shards[i].m)+ for _, g := range m.shards[i].ring {+ held += len(g.values)+ }+ }+ if held != 0 {+ t.Errorf("%d long values held after the only one was replaced", held)}- if total > shards*100 {- t.Errorf("store holds %d keys, limit %d", total, shards*100)+}++// A key is forgotten before it expires only when live keys fill the store: in a steady state of+// keys written at a constant rate and expiring a TTL later, with the live ones at 70% of the limit,+// every key written within the last TTL and never read since is still held.+func TestKeepsLiveKeysUntilFull(t *testing.T) {+ const (+ limit = shards * 2000+ ttl = 1000 // ms+ live = limit * 7 / 10+ )+ m := NewMemory(limit)+ start := time.Unix(1000, 0)+ now := start+ m.now = func() time.Time { return now }+ ctx := context.Background()+ written := 0+ for ms := range 4 * ttl {+ now = start.Add(time.Duration(ms) * time.Millisecond)+ for ; written < (ms+1)*live/ttl; written++ {+ _, _ = m.Add(ctx, Entry{fmt.Sprint("k", written), []byte{1}, ttl * time.Millisecond})+ }+ }+ lost, from := 0, written-live+live/ttl // the keys written within the last TTL+ for i := from; i < written; i++ {+ if vals, _ := m.Get(ctx, fmt.Sprint("k", i)); vals[0] == nil {+ lost+++ }+ }+ if lost > 0 {+ t.Errorf("%d of %d live keys forgotten with live keys at 70%% of the limit", lost, written-from)+ }+}++// A key's expiry is kept to the millisecond, rounded up, so a key outlives its TTL by less than a+// millisecond and never expires before it, wherever in a millisecond it was written.+func TestShortTTL(t *testing.T) {+ m := NewMemory(1000)+ written := time.Unix(1000, 900_000) // 0.9 ms into a millisecond+ now := written+ m.now = func() time.Time { return now }+ ctx := context.Background()+ _ = m.Set(ctx, Entry{"k", []byte("v"), time.Millisecond}) // expires 1.9 ms into it+ now = written.Add(950 * time.Microsecond)+ if vals, _ := m.Get(ctx, "k"); vals[0] == nil {+ t.Error("key expired before its TTL")+ }+ now = written.Add(2 * time.Millisecond)+ if vals, _ := m.Get(ctx, "k"); vals[0] != nil {+ t.Error("key outlived its TTL by a millisecond or more")}}