Skip to content

Agent Mesh

17.6 KB
Raw Blame
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/agentmesh/api"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

const (
	provideFailureScopeIngress = "ingress"
	provideFailureScopeName    = "name"
	provideFailureScopeType    = "type"
)

var serviceProvideFailuresTotal = promauto.NewCounterVec(
	prometheus.CounterOpts{
		Name: "agentmesh_node_service_provide_failures_total",
		Help: "Failed directory announcements for local services, by announcement scope",
	},
	[]string{"scope"},
)

func init() {
	for _, scope := range []string{provideFailureScopeIngress, provideFailureScopeName, provideFailureScopeType} {
		serviceProvideFailuresTotal.WithLabelValues(scope)
	}
}

// directoryProvider is what ServiceRegistry asks of the directory: to
// announce a key and to withdraw it. Production wires this to the node's
// Directory; tests use a fake.
type directoryProvider interface {
	Provide(ctx context.Context, key []byte, service string) error
	Withdraw(ctx context.Context, key []byte) error
}

type ingressDirectoryProvider interface {
	ProvideWithIngress(ctx context.Context, key []byte, service string, ingress *api.ServiceIngress) error
}

// backendProber is implemented by services that can be asked whether their
// backend is really serving. Services that cannot be asked are advertised
// unconditionally.
type backendProber interface {
	Probe(ctx context.Context) error
}

// defaultDHTProbeTimeout is the default bound on one backend probe before
// advertising, used when a ServiceRegistry isn't given an explicit
// BackendProbeTimeout (see Options.BackendProbeTimeout / --backend-probe-
// timeout). Command-spawned backends (agentmesh-node.yaml's `command`, launched as
// a local subprocess) can need longer than this to answer their first
// request - a moderately-featured interpreted-language MCP server's own
// import/startup cost alone can exceed 2s - so this is a floor, not
// something every backend is expected to meet.
const defaultDHTProbeTimeout = 2 * time.Second

// advertisable reports whether a service is fit to be published to the DHT.
//
// Gossip already withholds announcements for a backend that does not answer,
// but the DHT is the other half of discovery and had no such check, so a
// backend that is not an MCP server at all stayed discoverable as one: it was
// listed by discover_remote_services and only failed later, at initialize, in
// the caller's face. Advertising is a claim the node makes on the backend's
// behalf, so it is the node that should verify it.
func advertisable(ctx context.Context, svc Service, probeTimeout time.Duration) error {
	prober, ok := svc.(backendProber)
	if !ok {
		return nil
	}
	probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
	defer cancel()
	return prober.Probe(probeCtx)
}

// UnannouncedService is an optional interface implemented by services (such as
// embedded meshnet listeners with NoAnnounce: true) that serve direct callers
// without publishing records to the router directory.
type UnannouncedService interface {
	NoAnnounce() bool
}

// IngressStateObserver is an optional interface notified whenever directory
// announcement updates whether external router ingress is active for the service.
type IngressStateObserver interface {
	SetIngressActive(active bool)
}

// ExternalIngressGate is an optional interface consulted by the external
// ingress server before forwarding a router-delivered request to the service.
type ExternalIngressGate interface {
	ExternalIngressAllowed() bool
}

// ServiceRegistry is the type-agnostic owner of registered services.
type ServiceRegistry struct {
	mu       sync.RWMutex
	services map[string]Service
	dht      directoryProvider

	// reprovideNow asks the node to run a reprovide cycle now, so a service
	// registered after the loop last ran does not wait a whole interval to be
	// advertised. Optional; nil outside a running node.
	reprovideNow func()

	// onIngressRegistered is called when an ingress-enabled service is
	// registered so the node can log its public router ingress URLs.
	onIngressRegistered func(info *api.ServiceInfo)

	// backendProbeTimeout bounds each backend probe in advertisable. Set once
	// at construction (see NewServiceRegistry); never mutated afterwards, so
	// reading it needs no lock.
	backendProbeTimeout time.Duration
}

// NewServiceRegistry constructs a registry bounding backend probes by
// backendProbeTimeout. A zero or negative value falls back to
// defaultDHTProbeTimeout, so callers can pass an unset
// Options.BackendProbeTimeout straight through without an explicit
// zero-check.
func NewServiceRegistry(d directoryProvider, backendProbeTimeout time.Duration) *ServiceRegistry {
	if backendProbeTimeout <= 0 {
		backendProbeTimeout = defaultDHTProbeTimeout
	}
	return &ServiceRegistry{
		services:            map[string]Service{},
		dht:                 d,
		backendProbeTimeout: backendProbeTimeout,
	}
}

// Register initialises a service, advertises it on the DHT, and inserts it
// into the map. Init runs before Provide so a failed handler-build never
// briefly advertises an unservable name.
//
// A backend that does not answer is registered but not advertised, rather than
// rejected: backends routinely start after the node does, and the reprovide
// loop picks them up once they answer.
//
// registering a name again replaces the instance and tears the old one down
func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {
	info := svc.Info()
	if info.Type == api.ServiceType_SERVICE_TYPE_UNSPECIFIED {
		return fmt.Errorf("cannot register service with unspecified type")
	}
	// The same check the config loader runs, for services that arrive by
	// other routes (FFI, tests): a name carrying its own scheme, such as
	// "plugin://x", would be addressed as type://plugin://x and dispatch
	// could no longer tell which type the policy was evaluated on.
	typeStr, err := api.ServiceTypeToString(info.Type)
	if err != nil {
		return err
	}
	if err := api.ValidateServiceFormat(typeStr + "://" + info.Name); err != nil {
		return fmt.Errorf("invalid service name %q: %w", info.Name, err)
	}
	if err := api.ValidateServiceIngress(info.Type, info.GetIngress()); err != nil {
		return fmt.Errorf("service %q: %w", info.Name, err)
	}

	// Policy is evaluated on type://name and the registry is keyed on name,
	// so a second type under the same name would let a grant for one reach
	// the other. One name, one service.
	r.mu.RLock()
	existing, taken := r.services[info.Name]
	r.mu.RUnlock()
	if taken && existing.Info().Type != info.Type {
		return fmt.Errorf("service name %q is already registered as %s; a name cannot serve two types", info.Name, existing.Info().Type)
	}

	if err := svc.Init(ctx); err != nil {
		return fmt.Errorf("init %s: %w", info.Name, err)
	}

	nameKey, err := api.ServiceKey(info.Type, info.Name)
	if err != nil {
		return err
	}
	typeKey, err := api.ServiceKey(info.Type, "")
	if err != nil {
		return err
	}

	skipAnnounce := isUnannouncedService(svc)
	var probeErr error
	ingressAnnounced := false
	if !skipAnnounce {
		probeErr = advertisable(ctx, svc, r.backendProbeTimeout)
		if probeErr != nil {
			logger.Warnf("[ServiceRegistry] Registered %s/%s but not advertising it: backend did not answer: %v", info.Type, info.Name, probeErr)
		} else {
			ingressAnnounced = r.announceServiceKeys(ctx, info, nameKey, typeKey)
		}
	}
	if obs, ok := svc.(IngressStateObserver); ok {
		obs.SetIngressActive(ingressAnnounced)
	}

	r.mu.Lock()
	replaced := r.services[info.Name]
	r.services[info.Name] = svc
	r.mu.Unlock()

	if replaced != nil && replaced != svc {
		if err := replaced.Teardown(); err != nil {
			logger.Errorf("[ServiceRegistry] Teardown replaced %s: %v", info.Name, err)
		}
	}

	if skipAnnounce {
		logger.Infof("[ServiceRegistry] Registered %s/%s (directory announcement disabled)", info.Type, info.Name)
	} else if probeErr == nil {
		logger.Infof("[ServiceRegistry] Registered and announced %s/%s", info.Type, info.Name)
		if ingressAnnounced && r.onIngressRegistered != nil {
			r.onIngressRegistered(info)
		}
	} else if r.reprovideNow != nil {
		r.reprovideNow()
	}
	return nil
}

func isUnannouncedService(svc Service) bool {
	u, ok := svc.(UnannouncedService)
	return ok && u.NoAnnounce()
}

// Unregister removes the service from the map, withdraws its records from
// the directory and calls Teardown. The record by type stays while another
// service of that type is registered. Unknown names are a no-op.
func (r *ServiceRegistry) Unregister(ctx context.Context, name string) error {
	r.mu.Lock()
	svc, ok := r.services[name]
	delete(r.services, name)
	typeStillServed := false
	if ok {
		for _, other := range r.services {
			if !isUnannouncedService(other) && other.Info().Type == svc.Info().Type {
				typeStillServed = true
				break
			}
		}
	}
	r.mu.Unlock()
	if !ok {
		return nil
	}
	info := svc.Info()
	if !isUnannouncedService(svc) {
		r.withdraw(ctx, info.Type, info.Name)
		if !typeStillServed {
			r.withdraw(ctx, info.Type, "")
		}
	}
	if err := svc.Teardown(); err != nil {
		logger.Errorf("[ServiceRegistry] Teardown %s: %v", name, err)
	}
	logger.Infof("[ServiceRegistry] Unregistered %s", name)
	return nil
}

// announceServiceKeys publishes the service's name and type directory keys.
// When an external ingress announcement is refused or fails, it logs an error,
// increments agentmesh_node_service_provide_failures_total{scope="ingress"},
// and immediately falls back to an internal-only announcement without ingress
// metadata so in-mesh discovery continues working. It returns true iff the
// service declared ingress and its ingress directory announcement succeeded.
func (r *ServiceRegistry) announceServiceKeys(ctx context.Context, info *api.ServiceInfo, nameKey, typeKey []byte) bool {
	target := serviceTarget(info)
	ing := info.GetIngress()
	ingressAnnounced := false

	if ing != nil {
		if ip, ok := r.dht.(ingressDirectoryProvider); ok {
			ingCtx, ingCancel := context.WithTimeout(ctx, 5*time.Second)
			err := ip.ProvideWithIngress(ingCtx, nameKey, target, ing)
			ingCancel()
			if err == nil {
				ingressAnnounced = true
			} else {
				if ctx.Err() == nil {
					serviceProvideFailuresTotal.WithLabelValues(provideFailureScopeIngress).Inc()
					logger.Errorf("[ServiceRegistry] External ingress announcement for %s/%s failed; falling back to internal-only directory announcement: %v", info.Type, info.Name, err)
				}
				nameCtx, nameCancel := context.WithTimeout(ctx, 5*time.Second)
				fallbackErr := r.dht.Provide(nameCtx, nameKey, target)
				nameCancel()
				if fallbackErr != nil && ctx.Err() == nil {
					serviceProvideFailuresTotal.WithLabelValues(provideFailureScopeName).Inc()
					logger.Errorf("[ServiceRegistry] Announcing %s/%s by name (internal fallback): %v", info.Type, info.Name, fallbackErr)
				}
			}
		} else {
			nameCtx, nameCancel := context.WithTimeout(ctx, 5*time.Second)
			err := r.dht.Provide(nameCtx, nameKey, target)
			nameCancel()
			if err == nil {
				ingressAnnounced = true
			} else if ctx.Err() == nil {
				serviceProvideFailuresTotal.WithLabelValues(provideFailureScopeName).Inc()
				logger.Errorf("[ServiceRegistry] Announcing %s/%s by name: %v", info.Type, info.Name, err)
			}
		}
	} else {
		nameCtx, nameCancel := context.WithTimeout(ctx, 5*time.Second)
		err := r.dht.Provide(nameCtx, nameKey, target)
		nameCancel()
		if err != nil && ctx.Err() == nil {
			serviceProvideFailuresTotal.WithLabelValues(provideFailureScopeName).Inc()
			logger.Errorf("[ServiceRegistry] Announcing %s/%s by name: %v", info.Type, info.Name, err)
		}
	}

	typeCtx, typeCancel := context.WithTimeout(ctx, 5*time.Second)
	typeErr := r.dht.Provide(typeCtx, typeKey, target)
	typeCancel()
	if typeErr != nil && ctx.Err() == nil {
		serviceProvideFailuresTotal.WithLabelValues(provideFailureScopeType).Inc()
		logger.Errorf("[ServiceRegistry] Announcing %s/%s by type: %v", info.Type, info.Name, typeErr)
	}

	return ingressAnnounced
}

// serviceTarget is a service as a record names it, "<type>://<name>".
func serviceTarget(info *api.ServiceInfo) string {
	typeStr, err := api.ServiceTypeToString(info.GetType())
	if err != nil {
		return ""
	}
	return typeStr + "://" + info.GetName()
}

// withdraw removes one of the registry's records from the directory. A
// failure is logged: the record expires on its own within the mesh's
// record TTL.
func (r *ServiceRegistry) withdraw(ctx context.Context, serviceType api.ServiceType, name string) {
	key, err := api.ServiceKey(serviceType, name)
	if err != nil {
		logger.Warnf("[ServiceRegistry] Invalid service key for %s/%s: %v", serviceType, name, err)
		return
	}
	withdrawCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	if err := r.dht.Withdraw(withdrawCtx, key); err != nil {
		if errors.Is(err, errNoDirectoryRouter) {
			logger.Debugf("[ServiceRegistry] Skipping withdraw of %s/%s: %v", serviceType, name, err)
		} else {
			logger.Warnf("[ServiceRegistry] Withdrawing %s/%s from the directory: %v", serviceType, name, err)
		}
	}
}

// WithdrawAll withdraws every registered service's records from the
// directory, by name and by type, as a node does before it leaves the mesh.
func (r *ServiceRegistry) WithdrawAll(ctx context.Context) {
	r.mu.RLock()
	infos := make([]*api.ServiceInfo, 0, len(r.services))
	for _, svc := range r.services {
		if isUnannouncedService(svc) {
			continue
		}
		infos = append(infos, svc.Info())
	}
	r.mu.RUnlock()
	types := map[api.ServiceType]bool{}
	for _, info := range infos {
		r.withdraw(ctx, info.Type, info.Name)
		types[info.Type] = true
	}
	for t := range types {
		r.withdraw(ctx, t, "")
	}
}

// Get returns the service registered under name, if any.
func (r *ServiceRegistry) Get(name string) (Service, bool) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	svc, ok := r.services[name]
	return svc, ok
}

// GetTyped returns the service registered under name only if it is of type
// t. Dispatch after authorization must use this: the policy was evaluated
// on t://name, and a service of another type under that name is not what
// the caller was granted.
func (r *ServiceRegistry) GetTyped(t api.ServiceType, name string) (Service, bool) {
	svc, ok := r.Get(name)
	if !ok || svc.Info().Type != t {
		return nil, false
	}
	return svc, true
}

// List returns the ServiceInfo for every registered service, optionally
// filtered by type. SERVICE_TYPE_UNSPECIFIED means "all types."
func (r *ServiceRegistry) List(typeFilter api.ServiceType) []*api.ServiceInfo {
	r.mu.RLock()
	defer r.mu.RUnlock()
	out := []*api.ServiceInfo{}
	for _, svc := range r.services {
		info := svc.Info()
		if typeFilter != api.ServiceType_SERVICE_TYPE_UNSPECIFIED && info.Type != typeFilter {
			continue
		}
		out = append(out, info)
	}
	return out
}

// insertService inserts a service into the registry without calling Init or
// advertising on the DHT. For tests only.
func (r *ServiceRegistry) insertService(svc Service) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.services[svc.Info().Name] = svc
}

// TeardownAll calls Teardown on every registered service and clears the
// map. Per-service errors are logged; iteration continues.
func (r *ServiceRegistry) TeardownAll() {
	r.mu.Lock()
	svcs := r.services
	r.services = map[string]Service{}
	r.mu.Unlock()
	for name, svc := range svcs {
		if err := svc.Teardown(); err != nil {
			logger.Errorf("[ServiceRegistry] Teardown %s: %v", name, err)
		}
	}
}

// ReprovideAll re-provides all registered services to the DHT concurrently and
// reports how many were withheld because their backend did not answer.
// A service whose backend has stopped answering falls out of the DHT by not
// being reprovided, and returns on a later tick once it answers again.
func (r *ServiceRegistry) ReprovideAll(ctx context.Context) int {
	r.mu.Lock()
	toProvide := make([]Service, 0, len(r.services))
	for _, svc := range r.services {
		toProvide = append(toProvide, svc)
	}
	r.mu.Unlock()

	var withheld atomic.Int32
	var wg sync.WaitGroup
	sem := make(chan struct{}, 5)

Loop:
	for _, svc := range toProvide {
		svc := svc
		select {
		case <-ctx.Done():
			break Loop
		case sem <- struct{}{}:
		}

		wg.Add(1)
		go func() {
			defer func() {
				<-sem
				wg.Done()
			}()

			if isUnannouncedService(svc) {
				return
			}
			info := svc.Info()
			if err := advertisable(ctx, svc, r.backendProbeTimeout); err != nil {
				withheld.Add(1)
				// On shutdown every service fails this way, and saying so
				// would blame backends for the node stopping.
				if ctx.Err() == nil {
					logger.Warnf("[ServiceRegistry] Withholding %s/%s from the directory: backend did not answer: %v", info.Type, info.Name, err)
				}
				return
			}

			nameKey, err := api.ServiceKey(info.Type, info.Name)
			if err != nil {
				logger.Warnf("[ServiceRegistry] Invalid service key for %s/%s: %v", info.Type, info.Name, err)
				return
			}
			typeKey, err := api.ServiceKey(info.Type, "")
			if err != nil {
				logger.Warnf("[ServiceRegistry] Invalid service type key for %s: %v", info.Type, err)
				return
			}
			ingressAnnounced := r.announceServiceKeys(ctx, info, nameKey, typeKey)
			if obs, ok := svc.(IngressStateObserver); ok {
				obs.SetIngressActive(ingressAnnounced)
			}
		}()
	}
	wg.Wait()
	return int(withheld.Load())
}