Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(gateway): return memory to the system once the gateway goes idle (#1276)

kyle-elliott-asymptote34e880bca64d
diff --git a/gateway/cmd/beacon-gateway/gc_idle.go b/gateway/cmd/beacon-gateway/gc_idle.go
new file mode 100644
index 00000000..683eda19
--- /dev/null
+++ b/gateway/cmd/beacon-gateway/gc_idle.go
@@ -0,0 +1,59 @@
+package main
+
+import (
+ "runtime"
+ rtdebug "runtime/debug"
+ "runtime/metrics"
+ "time"
+)
+
+// idleRelease is how long the gateway goes without a collection before it returns the memory its
+// last busy spell grew to. The collector runs only as the heap grows, and the runtime gives free
+// memory back to the system slowly, so after a burst an idle sidecar kept its peak footprint: 20
+// seconds after load, RSS was still 95% of it.
+const idleRelease = 30 * time.Second
+
+// startIdleRelease returns memory to the system once the gateway has gone idleRelease without a
+// collection, and again after each busy spell.
+func startIdleRelease() {
+ sample := []metrics.Sample{{Name: "/gc/cycles/total:gc-cycles"}}
+ cycles := func() uint64 {
+ metrics.Read(sample)
+ return sample[0].Value.Uint64()
+ }
+ go func() {
+ r := idleReleaser{last: cycles(), released: true} // nothing to give back before a busy spell
+ for range time.Tick(idleRelease) {
+ if r.idle(cycles()) {
+ // A sync.Pool keeps what was put back since the last collection through the
+ // next one, and the calls that ended the busy spell put their body buffers back
+ // after it: one collection first, so FreeOSMemory's own frees them.
+ runtime.GC()
+ rtdebug.FreeOSMemory()
+ r.didRelease(cycles())
+ }
+ }
+ }()
+}
+
+// idleReleaser decides when to release memory: at the first check after a whole interval with no
+// collection, and not again until a collection other than its own has run. A collection means the
+// gateway is allocating; while it is, memory released would only be taken back.
+type idleReleaser struct {
+ last uint64 // collections counted at the last check
+ released bool // released since the last collection
+}
+
+// idle reports, given the collections counted now, whether to release memory.
+func (r *idleReleaser) idle(cycles uint64) bool {
+ if cycles != r.last {
+ r.last, r.released = cycles, false
+ return false
+ }
+ return !r.released
+}
+
+// didRelease records a release and the collections counted after it, its own included.
+func (r *idleReleaser) didRelease(cycles uint64) {
+ r.last, r.released = cycles, true
+}
diff --git a/gateway/cmd/beacon-gateway/gc_idle_test.go b/gateway/cmd/beacon-gateway/gc_idle_test.go
new file mode 100644
index 00000000..198696f1
--- /dev/null
+++ b/gateway/cmd/beacon-gateway/gc_idle_test.go
@@ -0,0 +1,29 @@
+package main
+
+import (
+ "slices"
+ "testing"
+)
+
+// Memory is released once per idle spell: at the first check after a whole interval with no
+// collection, never while collections keep coming, and not again until the gateway has been busy;
+// the collection the release itself runs is not being busy.
+func TestIdleRelease(t *testing.T) {
+ // Collections that ran in each interval before a check.
+ activity := []int{0, 3, 5, 0, 0, 0, 1, 0, 0, 2, 0}
+ want := []int{3, 7, 10} // the checks that release
+ r := idleReleaser{released: true}
+ var gcs uint64
+ var got []int
+ for i, n := range activity {
+ gcs += uint64(n)
+ if r.idle(gcs) {
+ got = append(got, i)
+ gcs++ // the release's own collection
+ r.didRelease(gcs)
+ }
+ }
+ if !slices.Equal(got, want) {
+ t.Errorf("released at checks %v, want %v", got, want)
+ }
+}
diff --git a/gateway/cmd/beacon-gateway/main.go b/gateway/cmd/beacon-gateway/main.go
index 5e6769fe..30ec01cd 100644
--- a/gateway/cmd/beacon-gateway/main.go
+++ b/gateway/cmd/beacon-gateway/main.go
@@ -70,6 +70,7 @@ func run() error {
if err != nil {
return err
}
+ startIdleRelease()
var bodyBytes int64
switch {
case cfg.Pipeline.MaxBodyBytes != "":