Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node, deploy: align DHT provider record TTL with the 15m default
Antonio Ojea3a7bb34c5365
diff --git a/.github/k8s/sam-router-template.yaml b/.github/k8s/sam-router-template.yamlindex 2b2aeca6..3f916732 100644--- a/.github/k8s/sam-router-template.yaml+++ b/.github/k8s/sam-router-template.yaml@@ -68,9 +68,6 @@ spec:- "--jwt-path=/var/run/secrets/tokens/sam-token"- "--keys-path=/data/router.key"- "--metrics-addr=0.0.0.0:9090"- # How long a provider record outlives its last reprovide (nodes reprovide- # every 5m); 0s is the library's 48h. Set per environment in deploy.yaml.- - "--dht-provider-addr-ttl=${DHT_PROVIDER_ADDR_TTL}"# /healthz answers as soon as the process is up; /readyz once the# router has enrolled and its libp2p host is online.startupProbe:diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yamlindex 25666177..cc571e7a 100644--- a/.github/workflows/deploy.yaml+++ b/.github/workflows/deploy.yaml@@ -369,10 +369,6 @@ jobs:VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }}VAR_CLUSTER_NAME: ${{ vars.CLUSTER_NAME }}- # Bananas deploys several times a day and every deploy replaces the- # canary pods; with the library's 48h a caller meets their dead- # provider records for two days. Hub keeps the default.- VAR_DHT_PROVIDER_ADDR_TTL: ${{ vars.ENV_NAME == 'bananas' && '1h' || '0s' }}run: |print_rollout_diagnostics() {local namespace="$1"@@ -405,10 +401,9 @@ jobs:export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}"export CLUSTER_REGION="${VAR_CLUSTER_REGION}"export CLUSTER_NAME="${VAR_CLUSTER_NAME}"- export DHT_PROVIDER_ADDR_TTL="${VAR_DHT_PROVIDER_ADDR_TTL}"envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-control-plane-template.yaml | kubectl apply -f -- envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG} ${DHT_PROVIDER_ADDR_TTL}' < .github/k8s/sam-router-template.yaml | kubectl apply -f -+ envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-router-template.yaml | kubectl apply -f -envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-console-template.yaml | kubectl apply -f -envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f -# Managed Prometheus ships with GKE; a cluster without it should not block the mesh rollout.@@ -498,7 +493,6 @@ jobs:VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }}VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }}- VAR_DHT_PROVIDER_ADDR_TTL: ${{ vars.ENV_NAME == 'bananas' && '1h' || '0s' }}run: |export ENV_NAME="${VAR_ENV_NAME}"export NAMESPACE="sam-${ENV_NAME}"@@ -506,7 +500,6 @@ jobs:export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}"export CLUSTER_REGION="${VAR_CLUSTER_REGION}"export ZONE="${CLUSTER_REGION}-a"- export DHT_PROVIDER_ADDR_TTL="${VAR_DHT_PROVIDER_ADDR_TTL}"echo "Retrieving Admin Token to generate Bootstrap Token..."ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d)@@ -579,8 +572,7 @@ jobs:--listen=/ip4/0.0.0.0/tcp/4501 \--listen=/ip4/0.0.0.0/udp/4501/quic-v1 \--external-addr=/ip4/${EXTERNAL_IP}/tcp/4501 \- --external-addr=/ip4/${EXTERNAL_IP}/udp/4501/quic-v1 \- --dht-provider-addr-ttl='"${DHT_PROVIDER_ADDR_TTL}"''+ --external-addr=/ip4/${EXTERNAL_IP}/udp/4501/quic-v1'- name: Deploy OpenClaw Canaryenv:diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.goindex c8203b42..4b31a014 100644--- a/cmd/sam-node/main.go+++ b/cmd/sam-node/main.go@@ -873,7 +873,7 @@ func main() {runCmd.Flags().StringVar(&tlsCertFlag, "tls-cert", "", "Path to TLS certificate for sidecar API")runCmd.Flags().StringVar(&tlsKeyFlag, "tls-key", "", "Path to TLS key for sidecar API")runCmd.Flags().StringVar(&tlsCAFlag, "tls-ca", "", "Path to TLS CA for sidecar API mTLS")- runCmd.Flags().DurationVar(&dhtProviderAddrTTLFlag, "dht-provider-addr-ttl", 0, "Time-To-Live for DHT provider addresses (0s uses library default)")+ runCmd.Flags().DurationVar(&dhtProviderAddrTTLFlag, "dht-provider-addr-ttl", node.DefaultDHTProviderAddrTTL, "How long a DHT provider record lives after its last announcement")runCmd.Flags().DurationVar(&dhtMaxRecordAgeFlag, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")runCmd.Flags().IntVar(&dhtLookupLimitFlag, "dht-lookup-limit", 0, "Maximum number of providers to query from the DHT (0 uses default 20)")runCmd.Flags().IntVar(&discoveryConcurrencyFlag, "discovery-concurrency", 0, "Max concurrent catalog fetches during discovery (0 uses default 10)")diff --git a/internal/node/node.go b/internal/node/node.goindex 78bb9ed8..60927a25 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -104,6 +104,12 @@ const (// Reprovide intervalReprovideInterval = 5 * time.Minute+ // DefaultDHTProviderAddrTTL is three of a node's 5-minute reprovides: when+ // ModeAuto promotes a dialable node to a DHT server, a peer that is gone+ // drops out of its provider store within this window while one late+ // reprovide does not.+ DefaultDHTProviderAddrTTL = 3 * ReprovideInterval+// reprovideRetryInterval is the first retry after a backend was withheld,// doubling up to ReprovideInterval. Backends commonly start after the node// does, and the full interval is far too long to wait for one of those.@@ -634,14 +640,11 @@ func (n *SamNode) Start(ctx context.Context) error {dht.Mode(dht.ModeAuto),dht.ProtocolPrefix("/sam"),}- var pmOpts []records.Option- if n.config.DHTProviderAddrTTL > 0 {- pmOpts = append(pmOpts, records.ProviderAddrTTL(n.config.DHTProviderAddrTTL))- pmOpts = append(pmOpts, records.ProvideValidity(n.config.DHTProviderAddrTTL))- }- if len(pmOpts) > 0 {- dhtOpts = append(dhtOpts, dht.ProviderManagerOpts(pmOpts...))+ providerTTL := n.config.DHTProviderAddrTTL+ if providerTTL <= 0 {+ providerTTL = DefaultDHTProviderAddrTTL}+ dhtOpts = append(dhtOpts, dht.ProviderManagerOpts(records.ProviderAddrTTL(providerTTL), records.ProvideValidity(providerTTL)))if n.config.DHTMaxRecordAge > 0 {dhtOpts = append(dhtOpts, dht.MaxRecordAge(n.config.DHTMaxRecordAge))}diff --git a/internal/node/options.go b/internal/node/options.goindex 2f97fad9..21ed650e 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -63,6 +63,8 @@ type Options struct {// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.BiscuitTimeout time.Duration// DHT Options+ // DHTProviderAddrTTL is how long a provider record lives after its last+ // announcement; 0 is DefaultDHTProviderAddrTTL.DHTProviderAddrTTL time.DurationDHTMaxRecordAge time.DurationDHTLookupLimit intdiff --git a/site/content/docs/reference/sam-node.md b/site/content/docs/reference/sam-node.mdindex 1131789c..04733fd3 100644--- a/site/content/docs/reference/sam-node.md+++ b/site/content/docs/reference/sam-node.md@@ -112,7 +112,8 @@ explains how to enroll.| `--discovery-interval` | `30s` | DHT discovery polling interval. || `--dht-lookup-limit` | `20` | Providers to query per DHT lookup. || `--discovery-concurrency` | `10` | Concurrent catalog fetches during discovery. |-| `--dht-provider-addr-ttl`, `--dht-max-record-age` | library defaults | DHT record lifetimes. |+| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |+| `--dht-max-record-age` | library default | DHT value record lifetime. || `--backend-probe-timeout` | `2s` | How long a service backend may take to answer before the node declines to advertise it. Raise it for subprocesses that start slowly. || `--secrets-dir` | `/etc/sam/secrets` | Directory holding the credentials that the control plane's [egress destinations](../policy/#egress-destinations) name, one file per credential name. The platform puts the files there; the node reads a file on every request to the destination. || `--control-plane-sync-interval` | `15m` | How often signing keys, the ban set, router addresses and the mesh policy are pulled from the control plane. Keep it well below the control plane's `--key-grace-period`; raise it on large meshes. |diff --git a/tests/e2e/canary_manifests.bats b/tests/e2e/canary_manifests.batsindex 7ab6a670..14da3c02 100644--- a/tests/e2e/canary_manifests.bats+++ b/tests/e2e/canary_manifests.bats@@ -24,7 +24,6 @@ load "lib/container_mesh.bash"export ENV_NAME="e2e"export NAMESPACE="default"export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"-export DHT_PROVIDER_ADDR_TTL="0s"setup_file() {if ! command -v kind >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1; then