Skip to content

Agent Mesh

node, deploy: align DHT provider record TTL with the 15m default

Antonio Ojea3a7bb34c5365
diff --git a/.github/k8s/sam-router-template.yaml b/.github/k8s/sam-router-template.yaml
index 2b2aeca6..3f916732 100644
--- a/.github/k8s/sam-router-template.yaml
+++ b/.github/k8s/sam-router-template.yaml
@@ -68,9 +68,6 @@ spec:
- "--jwt-path=/var/run/secrets/tokens/sam-token"
- "--keys-path=/data/router.key"
- "--metrics-addr=0.0.0.0:9090"
- # How long a provider record outlives its last reprovide (nodes reprovide
- # every 5m); 0s is the library's 48h. Set per environment in deploy.yaml.
- - "--dht-provider-addr-ttl=${DHT_PROVIDER_ADDR_TTL}"
# /healthz answers as soon as the process is up; /readyz once the
# router has enrolled and its libp2p host is online.
startupProbe:
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 25666177..cc571e7a 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -369,10 +369,6 @@ jobs:
VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }}
VAR_CLUSTER_NAME: ${{ vars.CLUSTER_NAME }}
- # Bananas deploys several times a day and every deploy replaces the
- # canary pods; with the library's 48h a caller meets their dead
- # provider records for two days. Hub keeps the default.
- VAR_DHT_PROVIDER_ADDR_TTL: ${{ vars.ENV_NAME == 'bananas' && '1h' || '0s' }}
run: |
print_rollout_diagnostics() {
local namespace="$1"
@@ -405,10 +401,9 @@ jobs:
export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}"
export CLUSTER_REGION="${VAR_CLUSTER_REGION}"
export CLUSTER_NAME="${VAR_CLUSTER_NAME}"
- export DHT_PROVIDER_ADDR_TTL="${VAR_DHT_PROVIDER_ADDR_TTL}"
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-control-plane-template.yaml | kubectl apply -f -
- envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG} ${DHT_PROVIDER_ADDR_TTL}' < .github/k8s/sam-router-template.yaml | kubectl apply -f -
+ envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-router-template.yaml | kubectl apply -f -
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-console-template.yaml | kubectl apply -f -
envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f -
# Managed Prometheus ships with GKE; a cluster without it should not block the mesh rollout.
@@ -498,7 +493,6 @@ jobs:
VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }}
VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }}
- VAR_DHT_PROVIDER_ADDR_TTL: ${{ vars.ENV_NAME == 'bananas' && '1h' || '0s' }}
run: |
export ENV_NAME="${VAR_ENV_NAME}"
export NAMESPACE="sam-${ENV_NAME}"
@@ -506,7 +500,6 @@ jobs:
export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}"
export CLUSTER_REGION="${VAR_CLUSTER_REGION}"
export ZONE="${CLUSTER_REGION}-a"
- export DHT_PROVIDER_ADDR_TTL="${VAR_DHT_PROVIDER_ADDR_TTL}"
echo "Retrieving Admin Token to generate Bootstrap Token..."
ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d)
@@ -579,8 +572,7 @@ jobs:
--listen=/ip4/0.0.0.0/tcp/4501 \
--listen=/ip4/0.0.0.0/udp/4501/quic-v1 \
--external-addr=/ip4/${EXTERNAL_IP}/tcp/4501 \
- --external-addr=/ip4/${EXTERNAL_IP}/udp/4501/quic-v1 \
- --dht-provider-addr-ttl='"${DHT_PROVIDER_ADDR_TTL}"''
+ --external-addr=/ip4/${EXTERNAL_IP}/udp/4501/quic-v1'
- name: Deploy OpenClaw Canary
env:
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index c8203b42..4b31a014 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -873,7 +873,7 @@ func main() {
runCmd.Flags().StringVar(&tlsCertFlag, "tls-cert", "", "Path to TLS certificate for sidecar API")
runCmd.Flags().StringVar(&tlsKeyFlag, "tls-key", "", "Path to TLS key for sidecar API")
runCmd.Flags().StringVar(&tlsCAFlag, "tls-ca", "", "Path to TLS CA for sidecar API mTLS")
- runCmd.Flags().DurationVar(&dhtProviderAddrTTLFlag, "dht-provider-addr-ttl", 0, "Time-To-Live for DHT provider addresses (0s uses library default)")
+ runCmd.Flags().DurationVar(&dhtProviderAddrTTLFlag, "dht-provider-addr-ttl", node.DefaultDHTProviderAddrTTL, "How long a DHT provider record lives after its last announcement")
runCmd.Flags().DurationVar(&dhtMaxRecordAgeFlag, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")
runCmd.Flags().IntVar(&dhtLookupLimitFlag, "dht-lookup-limit", 0, "Maximum number of providers to query from the DHT (0 uses default 20)")
runCmd.Flags().IntVar(&discoveryConcurrencyFlag, "discovery-concurrency", 0, "Max concurrent catalog fetches during discovery (0 uses default 10)")
diff --git a/internal/node/node.go b/internal/node/node.go
index 78bb9ed8..60927a25 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -104,6 +104,12 @@ const (
// Reprovide interval
ReprovideInterval = 5 * time.Minute
+ // DefaultDHTProviderAddrTTL is three of a node's 5-minute reprovides: when
+ // ModeAuto promotes a dialable node to a DHT server, a peer that is gone
+ // drops out of its provider store within this window while one late
+ // reprovide does not.
+ DefaultDHTProviderAddrTTL = 3 * ReprovideInterval
+
// reprovideRetryInterval is the first retry after a backend was withheld,
// doubling up to ReprovideInterval. Backends commonly start after the node
// does, and the full interval is far too long to wait for one of those.
@@ -634,14 +640,11 @@ func (n *SamNode) Start(ctx context.Context) error {
dht.Mode(dht.ModeAuto),
dht.ProtocolPrefix("/sam"),
}
- var pmOpts []records.Option
- if n.config.DHTProviderAddrTTL > 0 {
- pmOpts = append(pmOpts, records.ProviderAddrTTL(n.config.DHTProviderAddrTTL))
- pmOpts = append(pmOpts, records.ProvideValidity(n.config.DHTProviderAddrTTL))
- }
- if len(pmOpts) > 0 {
- dhtOpts = append(dhtOpts, dht.ProviderManagerOpts(pmOpts...))
+ providerTTL := n.config.DHTProviderAddrTTL
+ if providerTTL <= 0 {
+ providerTTL = DefaultDHTProviderAddrTTL
}
+ dhtOpts = append(dhtOpts, dht.ProviderManagerOpts(records.ProviderAddrTTL(providerTTL), records.ProvideValidity(providerTTL)))
if n.config.DHTMaxRecordAge > 0 {
dhtOpts = append(dhtOpts, dht.MaxRecordAge(n.config.DHTMaxRecordAge))
}
diff --git a/internal/node/options.go b/internal/node/options.go
index 2f97fad9..21ed650e 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -63,6 +63,8 @@ type Options struct {
// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.
BiscuitTimeout time.Duration
// DHT Options
+ // DHTProviderAddrTTL is how long a provider record lives after its last
+ // announcement; 0 is DefaultDHTProviderAddrTTL.
DHTProviderAddrTTL time.Duration
DHTMaxRecordAge time.Duration
DHTLookupLimit int
diff --git a/site/content/docs/reference/sam-node.md b/site/content/docs/reference/sam-node.md
index 1131789c..04733fd3 100644
--- a/site/content/docs/reference/sam-node.md
+++ b/site/content/docs/reference/sam-node.md
@@ -112,7 +112,8 @@ explains how to enroll.
| `--discovery-interval` | `30s` | DHT discovery polling interval. |
| `--dht-lookup-limit` | `20` | Providers to query per DHT lookup. |
| `--discovery-concurrency` | `10` | Concurrent catalog fetches during discovery. |
-| `--dht-provider-addr-ttl`, `--dht-max-record-age` | library defaults | DHT record lifetimes. |
+| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |
+| `--dht-max-record-age` | library default | DHT value record lifetime. |
| `--backend-probe-timeout` | `2s` | How long a service backend may take to answer before the node declines to advertise it. Raise it for subprocesses that start slowly. |
| `--secrets-dir` | `/etc/sam/secrets` | Directory holding the credentials that the control plane's [egress destinations](../policy/#egress-destinations) name, one file per credential name. The platform puts the files there; the node reads a file on every request to the destination. |
| `--control-plane-sync-interval` | `15m` | How often signing keys, the ban set, router addresses and the mesh policy are pulled from the control plane. Keep it well below the control plane's `--key-grace-period`; raise it on large meshes. |
diff --git a/tests/e2e/canary_manifests.bats b/tests/e2e/canary_manifests.bats
index 7ab6a670..14da3c02 100644
--- a/tests/e2e/canary_manifests.bats
+++ b/tests/e2e/canary_manifests.bats
@@ -24,7 +24,6 @@ load "lib/container_mesh.bash"
export ENV_NAME="e2e"
export NAMESPACE="default"
export CANARY_NAMESPACE="sam-canary-${ENV_NAME}"
-export DHT_PROVIDER_ADDR_TTL="0s"
setup_file() {
if ! command -v kind >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1; then