Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(gateway): keep the JSONL output's written pages out of the container's memory (#1278)

kyle-elliott-asymptoteda263c7f207d
diff --git a/docs/runtimes/llm-gateway.mdx b/docs/runtimes/llm-gateway.mdx
index 0ec5ea99..381765b2 100644
--- a/docs/runtimes/llm-gateway.mdx
+++ b/docs/runtimes/llm-gateway.mdx
@@ -72,6 +72,14 @@ These apply whichever gateway you attach to.
goes through unrecorded. `gc.memory_limit` defaults to 80% of the container's memory limit.
- **Outputs.** Write `beacon.event` records to a file your log shipper reads. The file uses the
runtime JSONL schema, so the same SIEM mappings apply.
+- **Memory.** Size the sidecar by `beacon-gateway`'s own memory. A container's memory usage
+ also counts the page cache of files written inside it, which Linux keeps until the container
+ runs short. On Linux the `jsonl` output tells the kernel to drop what it has written once it is
+ on disk, so its file adds a few tens of MB rather than filling the container to its limit.
+ Another file or a log written in the same container fills it the same way: send it to a log
+ shipper or another container, or read the container's working set rather than its usage.
+ After a busy spell the gateway returns its memory to the system once it has been idle for
+ about 30 seconds.
- **Sessions.** Clients that send a session header get one session per conversation. Claude Code
sends `x-claude-code-session-id`; other clients can send `session_id`, `x-session-id` or
`conversation_id`, or set `pipeline.session_headers`.
diff --git a/gateway/go.mod b/gateway/go.mod
index 5ce6860d..08283067 100644
--- a/gateway/go.mod
+++ b/gateway/go.mod
@@ -5,6 +5,7 @@ go 1.27.0
require (
github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve v0.0.0
github.com/envoyproxy/go-control-plane/envoy v1.40.0
+ golang.org/x/sys v0.48.0
google.golang.org/grpc v1.85.0
gopkg.in/yaml.v3 v3.0.1
)
@@ -19,7 +20,6 @@ require (
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect
golang.org/x/net v0.60.0 // indirect
- golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect
diff --git a/gateway/sink/jsonl.go b/gateway/sink/jsonl.go
index f291a317..780a6750 100644
--- a/gateway/sink/jsonl.go
+++ b/gateway/sink/jsonl.go
@@ -40,8 +40,17 @@ type JSONL struct {
// unfinished reports that a failed batch could not be cut back, so the file may end in part
// of a line, which the next batch cuts off before writing.
unfinished bool
+ // writeFrom is where what has not been handed to the disk starts (releasePages): the file's
+ // pages before it are dropped from memory as they are written out.
+ writeFrom int64
}
+// releaseEvery is how much is written between page-cache releases. The kernel keeps a file's
+// written pages in memory, charged to the container, until it runs short; written steadily, they
+// filled a sidecar's memory limit and kept it there, with the kernel reclaiming on every
+// allocation. Released every 8 MiB, the output holds about two of those in memory.
+const releaseEvery = 8 << 20
+
// OpenJSONL opens path for appending, creating it with owner-only permissions: records carry
// prompts and tool output.
func OpenJSONL(path string) (*JSONL, error) {
@@ -110,7 +119,7 @@ func (s *JSONL) Write(_ context.Context, batch []pipeline.Record) error {
return &pipeline.WriteError{Dropped: len(batch), Err: err}
}
var skipped error
- written := 0
+ written, n := 0, 0
for _, r := range batch {
line, err := json.Marshal(r.Value)
if err != nil {
@@ -119,6 +128,7 @@ func (s *JSONL) Write(_ context.Context, batch []pipeline.Record) error {
}
s.w.Write(line)
s.w.WriteByte('\n')
+ n += len(line) + 1
written++
}
if err := s.w.Flush(); err != nil {
@@ -126,12 +136,28 @@ func (s *JSONL) Write(_ context.Context, batch []pipeline.Record) error {
s.unfinished = s.cutBack(start) != nil
return &pipeline.WriteError{Dropped: len(batch), Err: err}
}
+ s.release(start + int64(n))
if skipped != nil {
return &pipeline.WriteError{Dropped: len(batch) - written, Err: skipped}
}
return nil
}
+// release hands what was written since the last release to the disk once it reaches
+// releaseEvery, without waiting, and drops from memory everything handed over before, which is on
+// disk by now (what is still being written is dropped by a later release). end is the file's size
+// after the batch; a file that shrank (rotated by copytruncate) starts over from its beginning.
+func (s *JSONL) release(end int64) {
+ if end < s.writeFrom {
+ s.writeFrom = 0
+ }
+ if end-s.writeFrom < releaseEvery {
+ return
+ }
+ releasePages(s.f, s.writeFrom, s.writeFrom, end-s.writeFrom)
+ s.writeFrom = end
+}
+
func (s *JSONL) size() (int64, error) {
info, err := s.f.Stat()
if err != nil {
diff --git a/gateway/sink/pagecache_linux.go b/gateway/sink/pagecache_linux.go
new file mode 100644
index 00000000..7042805b
--- /dev/null
+++ b/gateway/sink/pagecache_linux.go
@@ -0,0 +1,32 @@
+//go:build linux
+
+package sink
+
+import (
+ "syscall"
+
+ "golang.org/x/sys/unix"
+)
+
+// releasePages tells the kernel it may drop f's first dropLen bytes from the page cache, all of it
+// handed to the disk by earlier calls (pages still being written are left alone, and caught by a
+// later call), and starts writing [writeOff, writeOff+writeLen) to disk without waiting for it.
+// Both are hints: errors are ignored, and a file without a descriptor is left to the kernel.
+func releasePages(f any, dropLen, writeOff, writeLen int64) {
+ sc, ok := f.(syscall.Conn)
+ if !ok {
+ return
+ }
+ rc, err := sc.SyscallConn()
+ if err != nil {
+ return
+ }
+ _ = rc.Control(func(fd uintptr) {
+ if dropLen > 0 {
+ _ = unix.Fadvise(int(fd), 0, dropLen, unix.FADV_DONTNEED)
+ }
+ if writeLen > 0 {
+ _ = unix.SyncFileRange(int(fd), writeOff, writeLen, unix.SYNC_FILE_RANGE_WRITE)
+ }
+ })
+}
diff --git a/gateway/sink/pagecache_other.go b/gateway/sink/pagecache_other.go
new file mode 100644
index 00000000..69ce19f1
--- /dev/null
+++ b/gateway/sink/pagecache_other.go
@@ -0,0 +1,6 @@
+//go:build !linux
+
+package sink
+
+// releasePages does nothing where the kernel offers no way to drop a file's written pages.
+func releasePages(any, int64, int64, int64) {}