Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
deploy: the DNS sync job reads a router's peer ID from its metrics, not its logs
Antonio Ojeaaed5278bba5e
diff --git a/.github/k8s/dns-sync-cronjob-template.yaml b/.github/k8s/dns-sync-cronjob-template.yamlindex b1f40549..9d4dc97a 100644--- a/.github/k8s/dns-sync-cronjob-template.yaml+++ b/.github/k8s/dns-sync-cronjob-template.yaml@@ -15,7 +15,7 @@ rules:resources: ["nodes"]verbs: ["get", "list", "watch"]- apiGroups: [""]- resources: ["pods", "pods/log"]+ resources: ["pods", "pods/proxy"]verbs: ["get", "list", "watch"]---apiVersion: rbac.authorization.k8s.io/v1@@ -51,21 +51,32 @@ spec:- -c- |set -e- # Extract IPs and PeerIDs of pods running sam-router+ # Publish every running router as bootstrap.<env> (A records)+ # and _dnsaddr.bootstrap.<env> (one TXT per listen address,+ # naming the router's peer ID). The peer ID is read from the+ # router's cluster-internal metrics endpoint through the API+ # server proxy, so it does not depend on log retention.echo "Finding sam-router pods in ${NAMESPACE}..."PODS=$(kubectl get pods -n ${NAMESPACE} -l app=sam-router-${ENV_NAME} -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.name}{" "}{.spec.nodeName}{"\n"}{end}')declare -a IP_ARRAY=()declare -a TXT_ARRAY=()+ # Set when a router's peer ID could not be read; the TXT set is+ # then left as it is, since rewriting it from a partial view+ # would drop routers that are still serving.+ TXT_INCOMPLETE=""while read -r pod node; doif [ -z "$pod" ]; then continue; fiecho "Checking pod: $pod on node: $node"IP_LIST=$(kubectl get node $node -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}')- POD_IP=$(kubectl get pod $pod -n ${NAMESPACE} -o jsonpath='{.status.podIP}')- # Get the PeerID by extracting it from the router pod logs- PEER_ID=$(kubectl logs $pod -n ${NAMESPACE} | grep -o 'PeerID: [a-zA-Z0-9]*' | head -n 1 | cut -d' ' -f2 || true)+ PEER_ID=$(kubectl get --raw "/api/v1/namespaces/${NAMESPACE}/pods/${pod}:9090/proxy/metrics" 2>/dev/null \+ | sed -n 's/^sam_router_info{peer_id="\([A-Za-z0-9]*\)"} 1$/\1/p' | head -n 1 || true)+ if [ -z "$PEER_ID" ]; then+ echo "No peer ID from $pod yet; leaving the TXT record set unchanged this run."+ TXT_INCOMPLETE=1+ fiif [ -n "$IP_LIST" ]; thenecho "Found IP $IP_LIST "@@ -94,6 +105,10 @@ spec:OLD_TXT=$(gcloud dns record-sets list --zone=${ENV_NAME} --project=${GCP_PROJECT_ID} --name="_dnsaddr.bootstrap.${ENV_NAME}.sam-mesh.dev." --type=TXT --format="value(rrdatas)" 2>/dev/null || true)OLD_TXT_CLEAN=$(echo $OLD_TXT | tr ';' '\n' | sort -u | tr '\n' ' ' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')+ if [ -n "$TXT_INCOMPLETE" ]; then+ UNIQUE_TXT="$OLD_TXT_CLEAN"+ fi+if [ "$UNIQUE_IPS" == "$OLD_IPS_CLEAN" ] && [ "$UNIQUE_TXT" == "$OLD_TXT_CLEAN" ]; thenecho "DNS records are already up to date. No changes needed."exit 0@@ -114,7 +129,7 @@ spec:gcloud dns record-sets transaction add --zone=${ENV_NAME} --project=${GCP_PROJECT_ID} --name="bootstrap.${ENV_NAME}.sam-mesh.dev." --type=A --ttl=300 $UNIQUE_IPS || true# Remove old TXT records if they exist- if [ -n "$OLD_TXT" ]; then+ if [ -n "$OLD_TXT" ] && [ -z "$TXT_INCOMPLETE" ]; then# OLD_TXT could have spaces inside quotes. gcloud transaction remove needs each whole quoted string as separate arg# Let's just remove using the raw semicolon-delimited values by translating ';' to spaces, but careful with inner quotes!# Since TXT records are quoted, tr ';' ' ' works fine as long as there are no semicolons inside the DNSADDR.@@ -123,7 +138,7 @@ spec:fi# Add new TXT records- if [ -n "$UNIQUE_TXT" ]; then+ if [ -n "$UNIQUE_TXT" ] && [ -z "$TXT_INCOMPLETE" ]; then# we just pass $UNIQUE_TXT as arguments (it's already quoted words separated by space)eval "gcloud dns record-sets transaction add --zone=${ENV_NAME} --project=${GCP_PROJECT_ID} --name=\"_dnsaddr.bootstrap.${ENV_NAME}.sam-mesh.dev.\" --type=TXT --ttl=300 $UNIQUE_TXT || true"fidiff --git a/internal/router/metrics.go b/internal/router/metrics.goindex b5ca189f..10cca0b0 100644--- a/internal/router/metrics.go+++ b/internal/router/metrics.go@@ -79,6 +79,7 @@ func init() {type routerStateCollector struct {r *Router+ infoDesc *prometheus.DescreadyDesc *prometheus.DescauthenticatedDesc *prometheus.DescconnectedDesc *prometheus.Desc@@ -90,6 +91,10 @@ type routerStateCollector struct {func newRouterStateCollector(r *Router) *routerStateCollector {return &routerStateCollector{r: r,+ infoDesc: prometheus.NewDesc(+ "sam_router_info",+ "The router's mesh identity; always 1",+ []string{"peer_id"}, nil),readyDesc: prometheus.NewDesc("sam_router_ready","1 once the router is enrolled and its libp2p host is online",@@ -119,7 +124,7 @@ func newRouterStateCollector(r *Router) *routerStateCollector {func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {for _, d := range []*prometheus.Desc{- c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc,+ c.infoDesc, c.readyDesc, c.authenticatedDesc, c.connectedDesc, c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc,} {ch <- d}@@ -134,6 +139,7 @@ func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {return}ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 1)+ ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, r.Host.ID().String())ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.authenticatedPeers)))ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(r.Host.Network().Peers())))ch <- prometheus.MustNewConstMetric(c.bannedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.bannedPeers)))diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.goindex 3b0f93a9..ce63eb25 100644--- a/internal/router/metrics_test.go+++ b/internal/router/metrics_test.go@@ -98,6 +98,7 @@ func TestRouterMetricsListener(t *testing.T) {_, body = getBody(t, base+"/metrics")for _, want := range []string{"sam_router_ready 1",+ `sam_router_info{peer_id="` + host.ID().String() + `"} 1`,"sam_router_authenticated_peers 2","sam_router_banned_peers 1","sam_router_connected_peers 0",