Skip to content

Agent Mesh

node, router, control plane: --pprof-addr serves Go runtime profiles

Antonio Ojeaa0d82cea885b
diff --git a/.github/k8s/agentmesh-node-everything-template.yaml b/.github/k8s/agentmesh-node-everything-template.yaml
index 903a72cb..5c52b021 100644
--- a/.github/k8s/agentmesh-node-everything-template.yaml
+++ b/.github/k8s/agentmesh-node-everything-template.yaml
@@ -57,6 +57,7 @@ spec:
- "--jwt-path=/var/run/secrets/tokens/agentmesh-token"
- "--bind-addr=127.0.0.1:8080"
- "--metrics-addr=0.0.0.0:9090"
+ - "--pprof-addr=127.0.0.1:6060"
ports:
- containerPort: 8080
- containerPort: 9090
diff --git a/.github/k8s/agentmesh-router-template.yaml b/.github/k8s/agentmesh-router-template.yaml
index 18fc55a7..42ba7124 100644
--- a/.github/k8s/agentmesh-router-template.yaml
+++ b/.github/k8s/agentmesh-router-template.yaml
@@ -75,6 +75,9 @@ spec:
- "--jwt-path=/var/run/secrets/tokens/agentmesh-token"
- "--keys-path=/data/router.key"
- "--metrics-addr=0.0.0.0:9090"
+ # Profiles on loopback only: `kubectl port-forward <pod> 6060` then
+ # `go tool pprof http://127.0.0.1:6060/debug/pprof/heap`.
+ - "--pprof-addr=127.0.0.1:6060"
- "--high-watermark=${ROUTER_HIGH_WATERMARK}"
# /healthz answers as soon as the process is up; /readyz once the
# router has enrolled and its libp2p host is online.
diff --git a/cmd/agentmesh-control-plane/main.go b/cmd/agentmesh-control-plane/main.go
index 87049ab1..de90171e 100644
--- a/cmd/agentmesh-control-plane/main.go
+++ b/cmd/agentmesh-control-plane/main.go
@@ -25,6 +25,7 @@ import (
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/controlplane"
+ "github.com/google/agentmesh/internal/pprofserver"
"github.com/google/agentmesh/internal/secrets"
"github.com/google/agentmesh/internal/storage"
"github.com/google/agentmesh/internal/version"
@@ -35,6 +36,7 @@ import (
var (
bindAddress string
+ pprofAddr string
dbDriver string
dbDSN string
dbDSNPath string
@@ -173,6 +175,14 @@ func main() {
if err := srv.Start(); err != nil {
logger.Fatalf("Failed to start control plane: %v", err)
}
+ if pprofAddr != "" {
+ pprofSrv, err := pprofserver.Serve(pprofAddr)
+ if err != nil {
+ logger.Fatalf("Failed to serve profiles on %s: %v", pprofAddr, err)
+ }
+ defer func() { _ = pprofSrv.Close() }()
+ logger.Infof("Profiles at http://%s/debug/pprof/", pprofSrv.Addr)
+ }
logger.Infof("Agent Mesh Control Plane Online on %s", bindAddress)
<-cmd.Context().Done()
@@ -180,6 +190,7 @@ func main() {
}
rootCmd.Flags().StringVar(&bindAddress, "bind-address", "0.0.0.0:8080", "Address to listen on for web API services")
+ rootCmd.Flags().StringVar(&pprofAddr, "pprof-addr", "", "Serve Go runtime profiles under /debug/pprof/ on this address (e.g. 127.0.0.1:6060); unauthenticated and a heap profile is a memory dump, so bind it to loopback and reach it through a port-forward; off by default")
rootCmd.PersistentFlags().StringVar(&dbDriver, "db-driver", "sqlite", "Database driver (sqlite or postgres)")
rootCmd.PersistentFlags().StringVar(&dbDSN, "db-dsn", "control-plane.db", "Database DSN/Connection URL (avoid for postgres: embeds a password; prefer --db-dsn-path or AGENTMESH_DB_DSN)")
rootCmd.PersistentFlags().StringVar(&dbDSNPath, "db-dsn-path", "", "Path to file containing the database DSN/Connection URL (overrides --db-dsn; or env AGENTMESH_DB_DSN)")
diff --git a/cmd/agentmesh-node/main.go b/cmd/agentmesh-node/main.go
index ecc663ee..f6c396da 100644
--- a/cmd/agentmesh-node/main.go
+++ b/cmd/agentmesh-node/main.go
@@ -30,6 +30,7 @@ import (
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/node"
+ "github.com/google/agentmesh/internal/pprofserver"
"github.com/google/agentmesh/internal/secrets"
"github.com/google/agentmesh/internal/version"
golog "github.com/ipfs/go-log/v2"
@@ -65,6 +66,7 @@ var (
controlPlanePublicKeyFlag string
bindAddrFlag string
metricsAddrFlag string
+ pprofAddrFlag string
socketPathFlag string
discoveryIntervalFlag string
listenAddrs []string
@@ -663,6 +665,14 @@ func main() {
_ = metricsSrv.Shutdown(shutdownCtx)
}()
}
+ if pprofAddrFlag != "" {
+ pprofSrv, err := pprofserver.Serve(pprofAddrFlag)
+ if err != nil {
+ logger.Fatalf("Failed to serve profiles on %s: %v", pprofAddrFlag, err)
+ }
+ defer func() { _ = pprofSrv.Close() }()
+ logger.Infof("Profiles at http://%s/debug/pprof/", pprofSrv.Addr)
+ }
fmt.Printf("Agent Mesh Node Online.\nPeerID: %s\nListening on: %v\n", meshNode.Host.ID(), meshNode.Host.Addrs())
@@ -862,6 +872,7 @@ func main() {
runCmd.Flags().StringVar(&bindAddrFlag, "bind-addr", "127.0.0.1:8080", "Local TCP address for the HTTP server (MCP and Sidecar API); pass an empty value to serve only on the Unix socket")
runCmd.Flags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket serving the same API, where the socket's owner-only permissions replace the API token (defaults to <data-dir>/"+node.DefaultSocketName+"; pass an empty value to disable)")
runCmd.Flags().StringVar(&metricsAddrFlag, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address without authentication (e.g. 0.0.0.0:9090), for scrapers and probes that hold no API token; off by default")
+ runCmd.Flags().StringVar(&pprofAddrFlag, "pprof-addr", "", "Serve Go runtime profiles under /debug/pprof/ on this address (e.g. 127.0.0.1:6060); unauthenticated and a heap profile is a memory dump, so bind it to loopback and reach it through a port-forward; off by default")
runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", node.DefaultDiscoveryInterval, "Delay before the first announcement of this node's services to the DHT, capped at 5s; they are re-announced every 5 minutes after that")
runCmd.Flags().DurationVar(&monitorBootstrapFlag, "monitor-bootstrap", 2*time.Minute, "Initial wait before monitoring router connection")
runCmd.Flags().DurationVar(&monitorCheckIntervalFlag, "monitor-interval", 1*time.Minute, "Interval for checking router connection")
diff --git a/cmd/agentmesh-router/main.go b/cmd/agentmesh-router/main.go
index b4d3c4ff..a1dec84b 100644
--- a/cmd/agentmesh-router/main.go
+++ b/cmd/agentmesh-router/main.go
@@ -23,6 +23,7 @@ import (
"time"
"github.com/google/agentmesh/api"
+ "github.com/google/agentmesh/internal/pprofserver"
"github.com/google/agentmesh/internal/router"
"github.com/google/agentmesh/internal/version"
golog "github.com/ipfs/go-log/v2"
@@ -49,6 +50,7 @@ var (
lowWaterMark int
highWaterMark int
metricsAddr string
+ pprofAddr string
relayLimitDuration time.Duration
relayLimitData router.ByteSize
relayMaxReservations int
@@ -124,6 +126,14 @@ func main() {
_ = r.Close()
logger.Fatalf("Failed to start router: %v", err)
}
+ if pprofAddr != "" {
+ pprofSrv, err := pprofserver.Serve(pprofAddr)
+ if err != nil {
+ logger.Fatalf("Failed to serve profiles on %s: %v", pprofAddr, err)
+ }
+ defer func() { _ = pprofSrv.Close() }()
+ logger.Infof("Profiles at http://%s/debug/pprof/", pprofSrv.Addr)
+ }
<-cmd.Context().Done()
},
@@ -148,6 +158,7 @@ func main() {
rootCmd.Flags().IntVar(&lowWaterMark, "low-watermark", 0, "Connections the router trims down to once it is over --high-watermark; 0 is 15% below the high watermark, so a trim sheds that share and no more")
rootCmd.Flags().IntVar(&highWaterMark, "high-watermark", router.DefaultHighWaterMark, "Connections the router accepts before it trims; every member holds one, so this is the size of the mesh")
rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")
+ rootCmd.Flags().StringVar(&pprofAddr, "pprof-addr", "", "Serve Go runtime profiles under /debug/pprof/ on this address (e.g. 127.0.0.1:6060); unauthenticated and a heap profile is a memory dump, so bind it to loopback and reach it through a port-forward; off by default")
rootCmd.Flags().DurationVar(&relayLimitDuration, "relay-limit-duration", router.DefaultRelayLimitDuration, "Lifetime of each relayed connection (0 = no limit)")
rootCmd.Flags().Var(&relayLimitData, "relay-limit-data", "Bytes relayed per direction on each relayed connection, e.g. 512MiB (0 = no limit)")
rootCmd.Flags().IntVar(&relayMaxReservations, "relay-max-reservations", 0, "Members that may hold a relay reservation, which is what makes a member reachable through this router; 0 is --high-watermark")
diff --git a/internal/pprofserver/pprofserver.go b/internal/pprofserver/pprofserver.go
new file mode 100644
index 00000000..e1e59e73
--- /dev/null
+++ b/internal/pprofserver/pprofserver.go
@@ -0,0 +1,58 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pprofserver serves Go's runtime profiles on a listener of their
+// own. A profile is how a memory or goroutine figure that stays high after
+// a load leaves is attributed to the code holding it; the metrics say that
+// it is held, not by what. The listener is unauthenticated and a CPU
+// profile occupies the process for its duration, so it is off unless an
+// operator names an address, and that address should be loopback or a
+// cluster-internal one reached through a port-forward.
+package pprofserver
+
+import (
+ "net"
+ "net/http"
+ "net/http/pprof"
+ "time"
+)
+
+// Handler serves the pprof index and every profile under /debug/pprof/ on
+// a mux of its own. Importing net/http/pprof also registers them on
+// http.DefaultServeMux; no mesh component serves that mux.
+func Handler() http.Handler {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/debug/pprof/", pprof.Index)
+ mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+ mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+ mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+ mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+ return mux
+}
+
+// Serve listens on addr and serves Handler until the returned server is
+// shut down.
+func Serve(addr string) (*http.Server, error) {
+ listener, err := net.Listen("tcp", addr)
+ if err != nil {
+ return nil, err
+ }
+ server := &http.Server{
+ Addr: listener.Addr().String(),
+ Handler: Handler(),
+ ReadHeaderTimeout: 10 * time.Second,
+ }
+ go func() { _ = server.Serve(listener) }()
+ return server, nil
+}
diff --git a/internal/pprofserver/pprofserver_test.go b/internal/pprofserver/pprofserver_test.go
new file mode 100644
index 00000000..8b0cf3ff
--- /dev/null
+++ b/internal/pprofserver/pprofserver_test.go
@@ -0,0 +1,51 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pprofserver
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+)
+
+// The listener answers the profile index and a heap profile.
+func TestServeAnswersProfilesOnItsOwnListener(t *testing.T) {
+ srv, err := Serve("127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = srv.Shutdown(context.Background()) }()
+
+ resp, err := http.Get("http://" + srv.Addr + "/debug/pprof/")
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, _ := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "heap") {
+ t.Fatalf("index: %d %q", resp.StatusCode, body)
+ }
+ resp, err = http.Get("http://" + srv.Addr + "/debug/pprof/heap?debug=1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, _ = io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "heap profile") {
+ t.Fatalf("heap: %d %.80q", resp.StatusCode, body)
+ }
+}
diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.md
index a2c2feed..9609146d 100644
--- a/site/content/docs/reference/agentmesh-node.md
+++ b/site/content/docs/reference/agentmesh-node.md
@@ -101,6 +101,7 @@ explains how to enroll.
| `--tls-cert`, `--tls-key` | | Serve the TCP listener over TLS. |
| `--tls-ca` | | Require client certificates signed by this CA (mTLS). Callers with a verified certificate need no token. |
| `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address, for scrapers. Do not expose it on shared networks. |
+| `--pprof-addr` | off | Serve Go runtime profiles under `/debug/pprof/` on this address. Unauthenticated, and a heap profile is a dump of process memory, which holds the node's key and the credentials it has seen: bind it to loopback. |
| `--daemonize` | `false` | Start detached, wait for the API to answer, print the endpoint, token file and log path, and exit. Safe to run repeatedly. Generates `api-token` if no token is configured. Refuses to run on a node that has never enrolled. |
### Mesh
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index c1c0b75f..dd67824f 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -25,6 +25,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban
| `--oidc-client-id` | first audience | OAuth client ID advertised on `/info`, for providers where it differs from the audience. |
| `--insecure-skip-tls-verify` | `false` | Skip TLS verification when fetching issuer metadata and keys. For a cluster issuer served with the cluster CA, or a local development issuer. |
| `--bind-address` | `0.0.0.0:8080` | HTTP listen address. |
+| `--pprof-addr` | off | Serve Go runtime profiles under `/debug/pprof/` on this address. Unauthenticated, and a heap profile is a dump of process memory, which holds the signing keys and the admin token: bind it to loopback, never to the API address. |
| `--db-driver` | `sqlite` | `sqlite` or `postgres`. |
| `--db-dsn` | `control-plane.db` | Database DSN. A PostgreSQL DSN contains a password, so the next two options are preferred for PostgreSQL. |
| `--db-dsn-path` | | File containing the DSN. Overrides `--db-dsn`. Can also be set with `AGENTMESH_DB_DSN`. |
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index fd4f159a..a03720fb 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -51,6 +51,7 @@ replica count.
| `--shutdown-lease-ttl` | `30s` | Sent in the router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. The default covers a pod that restarts in place. A router that is being removed for good can send `1s`. `0` sends nothing and the lease expires on the control plane's schedule. |
| `--auth-deadline` | `1m` | How long an inbound connection may stay open without its peer passing the handshake. A member handshakes the moment it connects; a member attached elsewhere whose DHT client came to query is done in less, as is the control plane's event publisher, which presents a credential of its own. What is left at the deadline is closed, so the connections a router holds are its members' and the watermarks mean members. `0` keeps every connection. |
| `--metrics-addr` | off | Serve `/metrics`, `/healthz`, `/readyz` and `/debug/peer` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. |
+| `--pprof-addr` | off | Serve Go runtime profiles under `/debug/pprof/` on this address, for attributing memory or goroutines the metrics show as held. Unauthenticated, and a heap profile is a dump of process memory, which holds the router's key: bind it to loopback and reach it with `kubectl port-forward`. |
| `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |
## What it does at run time