Skip to content

Agent Mesh

fix service registration race

Antonio Ojea0cc69e056c3e
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index a28ae309..b69f4b98 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -227,6 +227,13 @@ func main() {
node.keysMu.Unlock()
}
+ // Register static services from config
+ if nodeConfig != nil && len(nodeConfig.Services) > 0 {
+ if err := node.RegisterStaticServices(context.Background(), nodeConfig.Services); err != nil {
+ logger.Errorf("Failed to register static services: %v", err)
+ }
+ }
+
// Start renewal loop
node.StartRenewalLoop(ctx, oidcIssuerFlag, clientIDFlag, clientSecretFlag, jwtPathFlag)
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 2b84e207..b5eff599 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -229,48 +229,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
node.startKeyPruning(ctx, keyGracePeriod)
// Register static services from config
- if nodeConfig != nil {
- var errs []error
- for _, sCfg := range nodeConfig.Services {
- sType, err := parseServiceType(sCfg.Type)
- if err != nil {
- logger.Errorf("[ServiceRegistry] Invalid service type %q for static service %s: %v", sCfg.Type, sCfg.Name, err)
- errs = append(errs, fmt.Errorf("invalid service type %q for static service %s: %w", sCfg.Type, sCfg.Name, err))
- continue
- }
-
- req := &api.RegisterServiceRequest{
- Service: &api.ServiceInfo{
- Type: sType,
- Name: sCfg.Name,
- Description: sCfg.Description,
- },
- }
-
- if sCfg.TargetURL != "" {
- req.Backend = &api.RegisterServiceRequest_TargetUrl{TargetUrl: sCfg.TargetURL}
- } else if len(sCfg.Command) > 0 {
- req.Backend = &api.RegisterServiceRequest_Command{
- Command: &api.CommandBackend{
- Command: sCfg.Command,
- Env: sCfg.Env,
- },
- }
- } else {
- logger.Errorf("[ServiceRegistry] Static service %s has no backend specified", sCfg.Name)
- errs = append(errs, fmt.Errorf("static service %s has no backend specified", sCfg.Name))
- continue
- }
- if err := node.RegisterService(ctx, req); err != nil {
- logger.Errorf("[ServiceRegistry] Failed to register static service %s: %v", sCfg.Name, err)
- errs = append(errs, fmt.Errorf("failed to register static service %s: %w", sCfg.Name, err))
- }
- }
- if len(errs) > 0 {
- return nil, fmt.Errorf("failed to register static services: %w", errors.Join(errs...))
- }
- }
// Start Ingress HTTP Server
if err := node.StartIngressServer(ctx); err != nil {
@@ -280,6 +239,50 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
return node, nil
}
+func (n *SamNode) RegisterStaticServices(ctx context.Context, services []api.ServiceConfig) error {
+ var errs []error
+ for _, sCfg := range services {
+ sType, err := parseServiceType(sCfg.Type)
+ if err != nil {
+ logger.Errorf("[ServiceRegistry] Invalid service type %q for static service %s: %v", sCfg.Type, sCfg.Name, err)
+ errs = append(errs, fmt.Errorf("invalid service type %q for static service %s: %w", sCfg.Type, sCfg.Name, err))
+ continue
+ }
+
+ req := &api.RegisterServiceRequest{
+ Service: &api.ServiceInfo{
+ Type: sType,
+ Name: sCfg.Name,
+ Description: sCfg.Description,
+ },
+ }
+
+ if sCfg.TargetURL != "" {
+ req.Backend = &api.RegisterServiceRequest_TargetUrl{TargetUrl: sCfg.TargetURL}
+ } else if len(sCfg.Command) > 0 {
+ req.Backend = &api.RegisterServiceRequest_Command{
+ Command: &api.CommandBackend{
+ Command: sCfg.Command,
+ Env: sCfg.Env,
+ },
+ }
+ } else {
+ logger.Errorf("[ServiceRegistry] Static service %s has no backend specified", sCfg.Name)
+ errs = append(errs, fmt.Errorf("static service %s has no backend specified", sCfg.Name))
+ continue
+ }
+
+ if err := n.RegisterService(ctx, req); err != nil {
+ logger.Errorf("[ServiceRegistry] Failed to register static service %s: %v", sCfg.Name, err)
+ errs = append(errs, fmt.Errorf("failed to register static service %s: %w", sCfg.Name, err))
+ }
+ }
+ if len(errs) > 0 {
+ return fmt.Errorf("failed to register static services: %w", errors.Join(errs...))
+ }
+ return nil
+}
+
func (n *SamNode) ConnectAndAuthWithHub(ctx context.Context, addr multiaddr.Multiaddr) error {
addrInfo, err := peer.AddrInfoFromP2pAddr(addr)
if err != nil {
diff --git a/cmd/sam-node/static_service_test.go b/cmd/sam-node/static_service_test.go
new file mode 100644
index 00000000..4c0b851b
--- /dev/null
+++ b/cmd/sam-node/static_service_test.go
@@ -0,0 +1,141 @@
+package main
+
+import (
+ "context"
+ "crypto/rand"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+ "google.golang.org/protobuf/proto"
+)
+
+func startMockLibp2pHub(t *testing.T) (peer.ID, string) {
+ t.Helper()
+ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatalf("failed to create mock libp2p host: %v", err)
+ }
+
+ kdht, err := dht.New(context.Background(), h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/sam"))
+ if err != nil {
+ t.Fatalf("failed to create DHT on mock hub: %v", err)
+ }
+
+ // Start HTTP server for enrollment
+ mux := http.NewServeMux()
+ mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ http.Error(w, "Failed to read body", http.StatusBadRequest)
+ return
+ }
+ var req api.EnrollRequest
+ if err := proto.Unmarshal(body, &req); err != nil {
+ http.Error(w, "Invalid request format", http.StatusBadRequest)
+ return
+ }
+
+ resp := &api.EnrollResponse{
+ BiscuitToken: []byte("mock-biscuit-token"),
+ HubPublicKey: []byte("mock-hub-pub-key"),
+ HubAddresses: []string{h.Addrs()[0].String() + "/p2p/" + h.ID().String()},
+ KnownPeers: []string{},
+ }
+ data, err := proto.Marshal(resp)
+ if err != nil {
+ http.Error(w, "Failed to marshal response", http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(data)
+ })
+
+ httpServer := httptest.NewServer(mux)
+
+ t.Cleanup(func() {
+ httpServer.Close()
+ _ = kdht.Close()
+ _ = h.Close()
+ })
+
+ return h.ID(), httpServer.URL
+}
+
+func TestStaticServiceRegistrationRequiresConnection(t *testing.T) {
+ // 1. Setup mock hub
+ _, hubURL := startMockLibp2pHub(t)
+
+ // 2. Create temp store
+ tmpDir := t.TempDir()
+ store, err := NewStore(tmpDir)
+ if err != nil {
+ t.Fatalf("failed to create store: %v", err)
+ }
+
+ // 3. Create dummy keys
+ priv, _, err := crypto.GenerateEd25519Key(rand.Reader)
+ if err != nil {
+ t.Fatalf("failed to generate libp2p key: %v", err)
+ }
+
+ // 4. Create Node with empty hub addrs initially
+ node, err := NewSamNode(context.Background(), priv, nil, []multiaddr.Multiaddr{}, store, "test-mesh", "100ms", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 24*time.Hour)
+ if err != nil {
+ t.Fatalf("failed to create node: %v", err)
+ }
+ defer func() {
+ if err := node.Host.Close(); err != nil {
+ t.Errorf("failed to close node host: %v", err)
+ }
+ }()
+
+ // 5. Try to register static services BEFORE connecting to hub
+ services := []api.ServiceConfig{
+ {
+ Type: "mcp",
+ Name: "test-service",
+ Description: "Test Service",
+ TargetURL: "http://localhost:8080",
+ },
+ }
+
+ ctx := context.Background()
+ err = node.RegisterStaticServices(ctx, services)
+ if err == nil {
+ t.Fatal("Expected RegisterStaticServices to fail before connecting to hub, but it succeeded")
+ }
+ if !strings.Contains(err.Error(), "failed to find any peer in table") {
+ t.Fatalf("Expected error to contain 'failed to find any peer in table', got: %v", err)
+ }
+
+ // 6. Now enroll and connect to hub (which sets up DHT and hub connection)
+ oldHubAddr := hubAddr
+ hubAddr = hubURL
+ defer func() { hubAddr = oldHubAddr }()
+
+ err = node.Enroll(ctx, "dummy-jwt")
+ if err != nil {
+ t.Fatalf("failed to enroll: %v", err)
+ }
+
+ // Now that we are connected, try to register again!
+ err = node.RegisterStaticServices(ctx, services)
+ if err != nil {
+ t.Fatalf("Expected RegisterStaticServices to succeed after enrollment, but failed: %v", err)
+ }
+}