Skip to content

Agent Mesh

9.1 KB
Raw Blame
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controlplane

import (
	"context"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"

	"github.com/google/agentmesh/internal/storage"
)

const (
	// DefaultNodeRetention keeps a lapsed enrollment for a month past its
	// session, and a bootstrap enrollment for a month past its last refresh,
	// long enough to be looked up when investigating an incident, and is
	// what --node-retention defaults to.
	DefaultNodeRetention = 30 * 24 * time.Hour

	// DefaultEphemeralNodeRetention is how long a member an ephemeral token
	// enrolled is kept after its last refresh: an hour, long past the few
	// minutes a member of a fleet, a probe or a CI job stays, and what
	// --ephemeral-node-retention defaults to.
	DefaultEphemeralNodeRetention = time.Hour

	// maxNodeGCInterval bounds how often the sweep runs. Rows lapse on a
	// scale of days, so nothing is gained by sweeping more often than hourly;
	// a retention far shorter than that, as a test sets, sweeps at a tenth
	// of it so a lapsed row does not outlive the retention by much.
	maxNodeGCInterval = time.Hour
	minNodeGCInterval = time.Second
)

// nodeGCInterval is the sweep period for a retention: a tenth of it, within
// [minNodeGCInterval, maxNodeGCInterval].
func nodeGCInterval(retention time.Duration) time.Duration {
	return min(maxNodeGCInterval, max(minNodeGCInterval, retention/10))
}

var nodesDeletedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
	Name: "agentmesh_control_plane_nodes_deleted_total",
	Help: "Enrolled node records removed by the node GC, by reason: session for a session that lapsed more than the retention ago, silent for a bootstrap member that did not refresh for the retention, ephemeral for one an ephemeral token enrolled that did not refresh for the ephemeral retention",
}, []string{"reason"})

const (
	deletedReasonSession   = "session"
	deletedReasonSilent    = "silent"
	deletedReasonEphemeral = "ephemeral"
)

var storageRowsDeletedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
	Name: "agentmesh_control_plane_storage_rows_deleted_total",
	Help: "Rows the storage GC removed from a table other than nodes: enrollment requests older than the retention, bootstrap tokens expired for the retention that no request refers to, router leases expired for the retention",
}, []string{"table"})

var catalogEntries = promauto.NewGauge(prometheus.GaugeOpts{
	Name: "agentmesh_control_plane_catalog_entries",
	Help: "Service reports the control plane holds in memory, one per node that reported since it was admitted",
})

const (
	tableEnrollmentRequests = "enrollment_requests"
	tableBootstrapTokens    = "bootstrap_tokens"
	tableRouters            = "routers"
)

func init() {
	for _, reason := range []string{deletedReasonSession, deletedReasonSilent, deletedReasonEphemeral} {
		nodesDeletedTotal.WithLabelValues(reason)
	}
	for _, table := range []string{tableEnrollmentRequests, tableBootstrapTokens, tableRouters} {
		storageRowsDeletedTotal.WithLabelValues(table)
	}
}

// runNodeGCLoop sweeps once at start and then hourly. Replicas each sweep
// against the shared database; the deletes are idempotent, so the race is
// harmless and not worth a claim.
func (s *Server) runNodeGCLoop() {
	defer s.wg.Done()
	s.gcStorage(time.Now())

	ticker := time.NewTicker(nodeGCInterval(s.config.NodeRetention))
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			s.gcStorage(time.Now())
		case <-s.ctx.Done():
			return
		}
	}
}

// gcStorage is one sweep over everything that grows with mesh activity: the
// node records, the enrollment requests, the bootstrap tokens, the router
// leases, and the in-memory service catalog. Each step logs its own failure
// and the next one still runs, so one table's trouble does not let another
// grow.
func (s *Server) gcStorage(now time.Time) {
	s.gcExpiredNodes(now)
	if s.ctx.Err() != nil {
		return
	}
	s.gcEnrollmentRecords(now)
	if s.ctx.Err() != nil {
		return
	}
	s.pruneCatalog(now)
}

// gcEnrollmentRecords deletes the enrollment requests resolved, or left
// pending, more than NodeRetention ago (EphemeralNodeRetention when an
// ephemeral token made them), then the bootstrap tokens that expired more
// than NodeRetention ago and that no remaining request refers to, then the
// router leases expired for as long. Requests go before tokens because a
// request refers to its token.
func (s *Server) gcEnrollmentRecords(now time.Time) {
	ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
	defer cancel()

	before := now.Add(-s.config.NodeRetention)
	requests, err := s.store.DeleteStaleEnrollmentRequests(ctx, before, now.Add(-s.config.EphemeralNodeRetention))
	if err != nil {
		logger.Errorf("Failed to delete stale enrollment requests: %v", err)
	} else if requests > 0 {
		storageRowsDeletedTotal.WithLabelValues(tableEnrollmentRequests).Add(float64(requests))
		logger.Infof("Deleted %d enrollment requests older than the retention", requests)
	}

	tokens, err := s.store.DeleteExpiredBootstrapTokens(ctx, before)
	if err != nil {
		logger.Errorf("Failed to delete expired bootstrap tokens: %v", err)
	} else if tokens > 0 {
		storageRowsDeletedTotal.WithLabelValues(tableBootstrapTokens).Add(float64(tokens))
		logger.Infof("Deleted %d bootstrap tokens expired more than %s ago", tokens, s.config.NodeRetention)
	}

	leases, err := s.store.DeleteExpiredRouterLeases(ctx, before)
	if err != nil {
		logger.Errorf("Failed to delete expired router leases: %v", err)
	} else if leases > 0 {
		storageRowsDeletedTotal.WithLabelValues(tableRouters).Add(float64(leases))
		logger.Infof("Deleted %d router leases expired more than %s ago", leases, s.config.NodeRetention)
	}
}

// pruneCatalog forgets the service report of every peer that is no longer
// an admitted node: its enrollment lapsed, was collected or was banned on
// another replica. The console already hides such reports; this keeps the
// cache from holding one entry per member that ever joined until the next
// restart.
func (s *Server) pruneCatalog(now time.Time) {
	ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
	defer cancel()

	snap := s.catalogSnapshot()
	if len(snap) == 0 {
		catalogEntries.Set(0)
		return
	}
	admitted, err := s.store.ListNodes(ctx, storage.NodeFilter{States: []storage.NodeState{storage.NodeAdmitted}, Now: now})
	if err != nil {
		logger.Errorf("Failed to list admitted nodes to prune the service catalog: %v", err)
		return
	}
	keep := make(map[string]struct{}, len(admitted))
	for i := range admitted {
		keep[admitted[i].PeerID] = struct{}{}
	}
	dropped := 0
	s.catalogMu.Lock()
	for peerID := range snap {
		if _, ok := keep[peerID]; !ok {
			delete(s.catalog, peerID)
			logger.Debugw("Dropped service report of a node no longer admitted", "peer_id", peerID)
			dropped++
		}
	}
	remaining := len(s.catalog)
	s.catalogMu.Unlock()
	catalogEntries.Set(float64(remaining))
	if dropped > 0 {
		logger.Infof("Dropped %d service reports of nodes no longer admitted, %d remain", dropped, remaining)
	}
}

// gcExpiredNodes deletes every unbanned node whose session expired more than
// NodeRetention before now, and every unbanned bootstrap member, which has
// no session, that did not refresh for NodeRetention, or for
// EphemeralNodeRetention when an ephemeral token enrolled it. Options.Validate
// has refused a retention that is not positive, so there is no "keep
// forever": an unbounded nodes table is what slowed every console refresh
// and metrics scrape.
func (s *Server) gcExpiredNodes(now time.Time) {
	ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
	defer cancel()

	deleted, err := s.store.DeleteExpiredNodes(ctx, now.Add(-s.config.NodeRetention))
	if err != nil {
		logger.Errorf("Failed to delete expired node records: %v", err)
		return
	}
	if deleted > 0 {
		nodesDeletedTotal.WithLabelValues(deletedReasonSession).Add(float64(deleted))
		logger.Infof("Deleted %d node records whose session expired more than %s ago", deleted, s.config.NodeRetention)
	}

	silent, ephemeral, err := s.store.DeleteSilentBootstrapNodes(ctx, now.Add(-s.config.NodeRetention), now.Add(-s.config.EphemeralNodeRetention))
	if err != nil {
		logger.Errorf("Failed to delete silent bootstrap node records: %v", err)
		return
	}
	if silent > 0 {
		nodesDeletedTotal.WithLabelValues(deletedReasonSilent).Add(float64(silent))
		logger.Infof("Deleted %d bootstrap node records that did not refresh for %s", silent, s.config.NodeRetention)
	}
	if ephemeral > 0 {
		nodesDeletedTotal.WithLabelValues(deletedReasonEphemeral).Add(float64(ephemeral))
		logger.Infof("Deleted %d ephemeral node records that did not refresh for %s", ephemeral, s.config.EphemeralNodeRetention)
	}
}