Skip to content

Agent Mesh

router: size the relay budget to the mesh

Antonio Ojea261b436c776f
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.go
index 7f2cd1cf..4b3b6df8 100644
--- a/cmd/sam-router/main.go
+++ b/cmd/sam-router/main.go
@@ -16,6 +16,7 @@ package main
import (
"context"
+ "fmt"
"os"
"os/signal"
"syscall"
@@ -49,6 +50,8 @@ var (
metricsAddr string
relayLimitDuration time.Duration
relayLimitData router.ByteSize
+ relayMaxReservations int
+ relayMaxCircuits int
shutdownLeaseTTL time.Duration
)
@@ -93,6 +96,8 @@ func main() {
MetricsAddr: metricsAddr,
RelayLimitDuration: relayLimitDuration,
RelayLimitData: int64(relayLimitData),
+ RelayMaxReservations: relayMaxReservations,
+ RelayMaxCircuits: relayMaxCircuits,
ShutdownLeaseTTL: shutdownLeaseTTL,
}
@@ -136,6 +141,8 @@ func main() {
rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")
rootCmd.Flags().DurationVar(&relayLimitDuration, "relay-limit-duration", router.DefaultRelayLimitDuration, "Lifetime of each relayed connection (0 = no limit)")
rootCmd.Flags().Var(&relayLimitData, "relay-limit-data", "Bytes relayed per direction on each relayed connection, e.g. 512MiB (0 = no limit)")
+ rootCmd.Flags().IntVar(&relayMaxReservations, "relay-max-reservations", 0, "Members that may hold a relay reservation, which is what makes a member reachable through this router; 0 is --high-watermark")
+ rootCmd.Flags().IntVar(&relayMaxCircuits, "relay-max-circuits", 0, fmt.Sprintf("Relayed connections one peer may hold at once through this router, as caller or as destination; 0 is %d", router.DefaultRelayMaxCircuits))
rootCmd.Flags().DurationVar(&shutdownLeaseTTL, "shutdown-lease-ttl", router.DefaultShutdownLeaseTTL, "Sent in this router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. 0 sends nothing and the lease expires on the control plane's schedule")
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
diff --git a/internal/router/config.go b/internal/router/config.go
index 3a539471..1d0b6f3f 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -54,6 +54,13 @@ type Options struct {
// no limit, so Default() leaves them alone.
RelayLimitDuration time.Duration
RelayLimitData int64
+ // RelayMaxReservations is how many members may hold a relay reservation,
+ // which is what makes a member reachable through this router. Zero is
+ // HighWaterMark: every connected member may be reachable.
+ RelayMaxReservations int
+ // RelayMaxCircuits is how many relayed connections one peer may hold at
+ // once through this router. Zero is DefaultRelayMaxCircuits.
+ RelayMaxCircuits int
// ShutdownLeaseTTL is the lease the router asks for as it stops: how
// long the control plane keeps listing it, which is the time the router
// expects to be away. Zero sends no last lease and the entry expires on
@@ -109,6 +116,12 @@ func (o *Options) Default() {
if o.ConnsPerSourceIP <= 0 {
o.ConnsPerSourceIP = DefaultConnsPerSourceIP(o.HighWaterMark)
}
+ if o.RelayMaxReservations <= 0 {
+ o.RelayMaxReservations = o.HighWaterMark
+ }
+ if o.RelayMaxCircuits <= 0 {
+ o.RelayMaxCircuits = DefaultRelayMaxCircuits
+ }
if o.KeysDBPath == "" {
o.KeysDBPath = "router.key"
}
diff --git a/internal/router/relay_limit.go b/internal/router/relay_limit.go
index 49c1dce5..70286e9b 100644
--- a/internal/router/relay_limit.go
+++ b/internal/router/relay_limit.go
@@ -26,6 +26,29 @@ import (
// DefaultRelayLimitDuration is how long a relayed connection lives unless the operator sets it.
const DefaultRelayLimitDuration = time.Hour
+// DefaultRelayMaxCircuits is how many relayed connections one peer may hold
+// open through this router at once, as source or as destination. libp2p's
+// own default is 16, sized for a peer that occasionally reaches a friend
+// behind a NAT; a service that is reachable only through the router serves
+// one caller per circuit, and a popular one serves hundreds.
+const DefaultRelayMaxCircuits = 1024
+
+// relayResources sizes the relay to the mesh. libp2p's defaults assume a
+// public DHT: 128 reservations in all, 32 per autonomous system and 8 per
+// address, so a fleet in one cloud region had its 33rd member unreachable.
+// A reservation is what makes a member reachable through the router, so the
+// budget follows the connection budget: every connected member may hold
+// one, as many per address as may connect from one, and the per-ASN cap is
+// not applied, since a whole cloud is one ASN.
+func relayResources(maxReservations, maxCircuits, perIP int) relay.Resources {
+ res := relay.DefaultResources()
+ res.MaxReservations = maxReservations
+ res.MaxReservationsPerASN = maxReservations
+ res.MaxReservationsPerIP = perIP
+ res.MaxCircuits = maxCircuits
+ return res
+}
+
// relayLimit maps operator limits (0 = none) onto go-libp2p, where only a nil
// limit is unlimited and a zero field cuts every circuit at once.
func relayLimit(duration time.Duration, data int64) *relay.RelayLimit {
diff --git a/internal/router/router.go b/internal/router/router.go
index 4d672284..49ed49ef 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -497,6 +497,7 @@ func (r *Router) Start() (retErr error) {
// Setup Relay
_, err = relay.New(hostNode, relay.WithACL(&relayACL{r: r}),
+ relay.WithResources(relayResources(r.config.RelayMaxReservations, r.config.RelayMaxCircuits, r.config.ConnsPerSourceIP)),
relay.WithLimit(relayLimit(r.config.RelayLimitDuration, r.config.RelayLimitData)))
if err != nil {
return err
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 67ed70d0..62fa1e80 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -45,6 +45,8 @@ replica count.
| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |
| `--dht-max-record-age` | library default | DHT value record lifetime. |
| `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. |
+| `--relay-max-reservations` | `--high-watermark` (`4000`) | Members that may hold a relay reservation on this router at once, which is how many members it can reach on behalf of callers. Per source address the cap is `--conns-per-source-ip`. |
+| `--relay-max-circuits` | `1024` | Relayed connections one peer may hold at once through this router, as caller or as destination. A service that many members call at once needs this many open circuits. |
| `--shutdown-lease-ttl` | `30s` | Sent in the router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. The default covers a pod that restarts in place. A router that is being removed for good can send `1s`. `0` sends nothing and the lease expires on the control plane's schedule. |
| `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. |
| `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |