Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node, router, control plane: --pprof-addr serves Go runtime profiles
Antonio Ojeaa0d82cea885b
diff --git a/.github/k8s/agentmesh-node-everything-template.yaml b/.github/k8s/agentmesh-node-everything-template.yamlindex 903a72cb..5c52b021 100644--- a/.github/k8s/agentmesh-node-everything-template.yaml+++ b/.github/k8s/agentmesh-node-everything-template.yaml@@ -57,6 +57,7 @@ spec:- "--jwt-path=/var/run/secrets/tokens/agentmesh-token"- "--bind-addr=127.0.0.1:8080"- "--metrics-addr=0.0.0.0:9090"+ - "--pprof-addr=127.0.0.1:6060"ports:- containerPort: 8080- containerPort: 9090diff --git a/.github/k8s/agentmesh-router-template.yaml b/.github/k8s/agentmesh-router-template.yamlindex 18fc55a7..42ba7124 100644--- a/.github/k8s/agentmesh-router-template.yaml+++ b/.github/k8s/agentmesh-router-template.yaml@@ -75,6 +75,9 @@ spec:- "--jwt-path=/var/run/secrets/tokens/agentmesh-token"- "--keys-path=/data/router.key"- "--metrics-addr=0.0.0.0:9090"+ # Profiles on loopback only: `kubectl port-forward <pod> 6060` then+ # `go tool pprof http://127.0.0.1:6060/debug/pprof/heap`.+ - "--pprof-addr=127.0.0.1:6060"- "--high-watermark=${ROUTER_HIGH_WATERMARK}"# /healthz answers as soon as the process is up; /readyz once the# router has enrolled and its libp2p host is online.diff --git a/cmd/agentmesh-control-plane/main.go b/cmd/agentmesh-control-plane/main.goindex 87049ab1..de90171e 100644--- a/cmd/agentmesh-control-plane/main.go+++ b/cmd/agentmesh-control-plane/main.go@@ -25,6 +25,7 @@ import ("github.com/google/agentmesh/api""github.com/google/agentmesh/internal/controlplane"+ "github.com/google/agentmesh/internal/pprofserver""github.com/google/agentmesh/internal/secrets""github.com/google/agentmesh/internal/storage""github.com/google/agentmesh/internal/version"@@ -35,6 +36,7 @@ import (var (bindAddress string+ pprofAddr stringdbDriver stringdbDSN stringdbDSNPath string@@ -173,6 +175,14 @@ func main() {if err := srv.Start(); err != nil {logger.Fatalf("Failed to start control plane: %v", err)}+ if pprofAddr != "" {+ pprofSrv, err := pprofserver.Serve(pprofAddr)+ if err != nil {+ logger.Fatalf("Failed to serve profiles on %s: %v", pprofAddr, err)+ }+ defer func() { _ = pprofSrv.Close() }()+ logger.Infof("Profiles at http://%s/debug/pprof/", pprofSrv.Addr)+ }logger.Infof("Agent Mesh Control Plane Online on %s", bindAddress)<-cmd.Context().Done()@@ -180,6 +190,7 @@ func main() {}rootCmd.Flags().StringVar(&bindAddress, "bind-address", "0.0.0.0:8080", "Address to listen on for web API services")+ rootCmd.Flags().StringVar(&pprofAddr, "pprof-addr", "", "Serve Go runtime profiles under /debug/pprof/ on this address (e.g. 127.0.0.1:6060); unauthenticated and a heap profile is a memory dump, so bind it to loopback and reach it through a port-forward; off by default")rootCmd.PersistentFlags().StringVar(&dbDriver, "db-driver", "sqlite", "Database driver (sqlite or postgres)")rootCmd.PersistentFlags().StringVar(&dbDSN, "db-dsn", "control-plane.db", "Database DSN/Connection URL (avoid for postgres: embeds a password; prefer --db-dsn-path or AGENTMESH_DB_DSN)")rootCmd.PersistentFlags().StringVar(&dbDSNPath, "db-dsn-path", "", "Path to file containing the database DSN/Connection URL (overrides --db-dsn; or env AGENTMESH_DB_DSN)")diff --git a/cmd/agentmesh-node/main.go b/cmd/agentmesh-node/main.goindex ecc663ee..f6c396da 100644--- a/cmd/agentmesh-node/main.go+++ b/cmd/agentmesh-node/main.go@@ -30,6 +30,7 @@ import ("github.com/google/agentmesh/api""github.com/google/agentmesh/internal/node"+ "github.com/google/agentmesh/internal/pprofserver""github.com/google/agentmesh/internal/secrets""github.com/google/agentmesh/internal/version"golog "github.com/ipfs/go-log/v2"@@ -65,6 +66,7 @@ var (controlPlanePublicKeyFlag stringbindAddrFlag stringmetricsAddrFlag string+ pprofAddrFlag stringsocketPathFlag stringdiscoveryIntervalFlag stringlistenAddrs []string@@ -663,6 +665,14 @@ func main() {_ = metricsSrv.Shutdown(shutdownCtx)}()}+ if pprofAddrFlag != "" {+ pprofSrv, err := pprofserver.Serve(pprofAddrFlag)+ if err != nil {+ logger.Fatalf("Failed to serve profiles on %s: %v", pprofAddrFlag, err)+ }+ defer func() { _ = pprofSrv.Close() }()+ logger.Infof("Profiles at http://%s/debug/pprof/", pprofSrv.Addr)+ }fmt.Printf("Agent Mesh Node Online.\nPeerID: %s\nListening on: %v\n", meshNode.Host.ID(), meshNode.Host.Addrs())@@ -862,6 +872,7 @@ func main() {runCmd.Flags().StringVar(&bindAddrFlag, "bind-addr", "127.0.0.1:8080", "Local TCP address for the HTTP server (MCP and Sidecar API); pass an empty value to serve only on the Unix socket")runCmd.Flags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket serving the same API, where the socket's owner-only permissions replace the API token (defaults to <data-dir>/"+node.DefaultSocketName+"; pass an empty value to disable)")runCmd.Flags().StringVar(&metricsAddrFlag, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address without authentication (e.g. 0.0.0.0:9090), for scrapers and probes that hold no API token; off by default")+ runCmd.Flags().StringVar(&pprofAddrFlag, "pprof-addr", "", "Serve Go runtime profiles under /debug/pprof/ on this address (e.g. 127.0.0.1:6060); unauthenticated and a heap profile is a memory dump, so bind it to loopback and reach it through a port-forward; off by default")runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", node.DefaultDiscoveryInterval, "Delay before the first announcement of this node's services to the DHT, capped at 5s; they are re-announced every 5 minutes after that")runCmd.Flags().DurationVar(&monitorBootstrapFlag, "monitor-bootstrap", 2*time.Minute, "Initial wait before monitoring router connection")runCmd.Flags().DurationVar(&monitorCheckIntervalFlag, "monitor-interval", 1*time.Minute, "Interval for checking router connection")diff --git a/cmd/agentmesh-router/main.go b/cmd/agentmesh-router/main.goindex b4d3c4ff..a1dec84b 100644--- a/cmd/agentmesh-router/main.go+++ b/cmd/agentmesh-router/main.go@@ -23,6 +23,7 @@ import ("time""github.com/google/agentmesh/api"+ "github.com/google/agentmesh/internal/pprofserver""github.com/google/agentmesh/internal/router""github.com/google/agentmesh/internal/version"golog "github.com/ipfs/go-log/v2"@@ -49,6 +50,7 @@ var (lowWaterMark inthighWaterMark intmetricsAddr string+ pprofAddr stringrelayLimitDuration time.DurationrelayLimitData router.ByteSizerelayMaxReservations int@@ -124,6 +126,14 @@ func main() {_ = r.Close()logger.Fatalf("Failed to start router: %v", err)}+ if pprofAddr != "" {+ pprofSrv, err := pprofserver.Serve(pprofAddr)+ if err != nil {+ logger.Fatalf("Failed to serve profiles on %s: %v", pprofAddr, err)+ }+ defer func() { _ = pprofSrv.Close() }()+ logger.Infof("Profiles at http://%s/debug/pprof/", pprofSrv.Addr)+ }<-cmd.Context().Done()},@@ -148,6 +158,7 @@ func main() {rootCmd.Flags().IntVar(&lowWaterMark, "low-watermark", 0, "Connections the router trims down to once it is over --high-watermark; 0 is 15% below the high watermark, so a trim sheds that share and no more")rootCmd.Flags().IntVar(&highWaterMark, "high-watermark", router.DefaultHighWaterMark, "Connections the router accepts before it trims; every member holds one, so this is the size of the mesh")rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")+ rootCmd.Flags().StringVar(&pprofAddr, "pprof-addr", "", "Serve Go runtime profiles under /debug/pprof/ on this address (e.g. 127.0.0.1:6060); unauthenticated and a heap profile is a memory dump, so bind it to loopback and reach it through a port-forward; off by default")rootCmd.Flags().DurationVar(&relayLimitDuration, "relay-limit-duration", router.DefaultRelayLimitDuration, "Lifetime of each relayed connection (0 = no limit)")rootCmd.Flags().Var(&relayLimitData, "relay-limit-data", "Bytes relayed per direction on each relayed connection, e.g. 512MiB (0 = no limit)")rootCmd.Flags().IntVar(&relayMaxReservations, "relay-max-reservations", 0, "Members that may hold a relay reservation, which is what makes a member reachable through this router; 0 is --high-watermark")diff --git a/internal/pprofserver/pprofserver.go b/internal/pprofserver/pprofserver.gonew file mode 100644index 00000000..e1e59e73--- /dev/null+++ b/internal/pprofserver/pprofserver.go@@ -0,0 +1,58 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++// Package pprofserver serves Go's runtime profiles on a listener of their+// own. A profile is how a memory or goroutine figure that stays high after+// a load leaves is attributed to the code holding it; the metrics say that+// it is held, not by what. The listener is unauthenticated and a CPU+// profile occupies the process for its duration, so it is off unless an+// operator names an address, and that address should be loopback or a+// cluster-internal one reached through a port-forward.+package pprofserver++import (+ "net"+ "net/http"+ "net/http/pprof"+ "time"+)++// Handler serves the pprof index and every profile under /debug/pprof/ on+// a mux of its own. Importing net/http/pprof also registers them on+// http.DefaultServeMux; no mesh component serves that mux.+func Handler() http.Handler {+ mux := http.NewServeMux()+ mux.HandleFunc("/debug/pprof/", pprof.Index)+ mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)+ mux.HandleFunc("/debug/pprof/profile", pprof.Profile)+ mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)+ mux.HandleFunc("/debug/pprof/trace", pprof.Trace)+ return mux+}++// Serve listens on addr and serves Handler until the returned server is+// shut down.+func Serve(addr string) (*http.Server, error) {+ listener, err := net.Listen("tcp", addr)+ if err != nil {+ return nil, err+ }+ server := &http.Server{+ Addr: listener.Addr().String(),+ Handler: Handler(),+ ReadHeaderTimeout: 10 * time.Second,+ }+ go func() { _ = server.Serve(listener) }()+ return server, nil+}diff --git a/internal/pprofserver/pprofserver_test.go b/internal/pprofserver/pprofserver_test.gonew file mode 100644index 00000000..8b0cf3ff--- /dev/null+++ b/internal/pprofserver/pprofserver_test.go@@ -0,0 +1,51 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package pprofserver++import (+ "context"+ "io"+ "net/http"+ "strings"+ "testing"+)++// The listener answers the profile index and a heap profile.+func TestServeAnswersProfilesOnItsOwnListener(t *testing.T) {+ srv, err := Serve("127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = srv.Shutdown(context.Background()) }()++ resp, err := http.Get("http://" + srv.Addr + "/debug/pprof/")+ if err != nil {+ t.Fatal(err)+ }+ body, _ := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "heap") {+ t.Fatalf("index: %d %q", resp.StatusCode, body)+ }+ resp, err = http.Get("http://" + srv.Addr + "/debug/pprof/heap?debug=1")+ if err != nil {+ t.Fatal(err)+ }+ body, _ = io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), "heap profile") {+ t.Fatalf("heap: %d %.80q", resp.StatusCode, body)+ }+}diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.mdindex a2c2feed..9609146d 100644--- a/site/content/docs/reference/agentmesh-node.md+++ b/site/content/docs/reference/agentmesh-node.md@@ -101,6 +101,7 @@ explains how to enroll.| `--tls-cert`, `--tls-key` | | Serve the TCP listener over TLS. || `--tls-ca` | | Require client certificates signed by this CA (mTLS). Callers with a verified certificate need no token. || `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address, for scrapers. Do not expose it on shared networks. |+| `--pprof-addr` | off | Serve Go runtime profiles under `/debug/pprof/` on this address. Unauthenticated, and a heap profile is a dump of process memory, which holds the node's key and the credentials it has seen: bind it to loopback. || `--daemonize` | `false` | Start detached, wait for the API to answer, print the endpoint, token file and log path, and exit. Safe to run repeatedly. Generates `api-token` if no token is configured. Refuses to run on a node that has never enrolled. |### Meshdiff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex c1c0b75f..dd67824f 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -25,6 +25,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban| `--oidc-client-id` | first audience | OAuth client ID advertised on `/info`, for providers where it differs from the audience. || `--insecure-skip-tls-verify` | `false` | Skip TLS verification when fetching issuer metadata and keys. For a cluster issuer served with the cluster CA, or a local development issuer. || `--bind-address` | `0.0.0.0:8080` | HTTP listen address. |+| `--pprof-addr` | off | Serve Go runtime profiles under `/debug/pprof/` on this address. Unauthenticated, and a heap profile is a dump of process memory, which holds the signing keys and the admin token: bind it to loopback, never to the API address. || `--db-driver` | `sqlite` | `sqlite` or `postgres`. || `--db-dsn` | `control-plane.db` | Database DSN. A PostgreSQL DSN contains a password, so the next two options are preferred for PostgreSQL. || `--db-dsn-path` | | File containing the DSN. Overrides `--db-dsn`. Can also be set with `AGENTMESH_DB_DSN`. |diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex fd4f159a..a03720fb 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -51,6 +51,7 @@ replica count.| `--shutdown-lease-ttl` | `30s` | Sent in the router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. The default covers a pod that restarts in place. A router that is being removed for good can send `1s`. `0` sends nothing and the lease expires on the control plane's schedule. || `--auth-deadline` | `1m` | How long an inbound connection may stay open without its peer passing the handshake. A member handshakes the moment it connects; a member attached elsewhere whose DHT client came to query is done in less, as is the control plane's event publisher, which presents a credential of its own. What is left at the deadline is closed, so the connections a router holds are its members' and the watermarks mean members. `0` keeps every connection. || `--metrics-addr` | off | Serve `/metrics`, `/healthz`, `/readyz` and `/debug/peer` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. |+| `--pprof-addr` | off | Serve Go runtime profiles under `/debug/pprof/` on this address, for attributing memory or goroutines the metrics show as held. Unauthenticated, and a heap profile is a dump of process memory, which holds the router's key: bind it to loopback and reach it with `kubectl port-forward`. || `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |## What it does at run time