Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
tests: watch the mesh, not the node's log
Antonio Ojeaedb1c1739f96
diff --git a/tests/integration/cuj_test.go b/tests/integration/cuj_test.godeleted file mode 100644index e27609b2..00000000--- a/tests/integration/cuj_test.go+++ /dev/null@@ -1,56 +0,0 @@-// Copyright 2026 Google LLC-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--package integration_test--import (- "context"- "os"- "path/filepath"- "strings"- "testing"- "time"-)--func TestSamNodeRunWithManualTokenStarts(t *testing.T) {- nodeBin := buildBinary(t, "./cmd/sam-node")- _, routerAddr := startMockRouter(t)- tmpHome := t.TempDir()- env := append(os.Environ(),- "HOME="+tmpHome,- "XDG_CONFIG_HOME="+filepath.Join(tmpHome, ".config"),- )-- stdout, stderr, err := runCommand(- t,- repoRoot(t),- 3*time.Second,- env,- "",- nodeBin,- "run", "--control-plane", routerAddr,- "--jwt", "test-jwt",- "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",- "--api-token-path", tokenPath(t, "dummy-token"),- )- if err != context.DeadlineExceeded {- t.Fatalf("expected run command to keep running until timeout, got: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)- }- out := stdout + stderr- if !strings.Contains(out, "SAM Node Online") {- t.Fatalf("node did not reach online state:\n%s", out)- }-}diff --git a/tests/integration/failover_test.go b/tests/integration/failover_test.goindex 6b05ea6d..a91c4460 100644--- a/tests/integration/failover_test.go+++ b/tests/integration/failover_test.go@@ -25,7 +25,6 @@ import ("os""os/exec""path/filepath"- "strings""sync""testing""time"@@ -150,6 +149,7 @@ roles: []"--keys-path", filepath.Join(tmpDir, "router_keysA.db"),"--allow-loopback","--oidc-token", routerJWT,+ "--lease-renew-interval", "1s",)if err := cmdRouterA.Start(); err != nil {t.Fatalf("failed to start Router A: %v", err)@@ -182,6 +182,7 @@ roles: []"--keys-path", filepath.Join(tmpDir, "router_keysB.db"),"--allow-loopback","--oidc-token", routerJWT,+ "--lease-renew-interval", "1s",)if err := cmdRouterB.Start(); err != nil {t.Fatalf("failed to start Router B: %v", err)@@ -202,10 +203,8 @@ roles: []"HOME="+nodeHome,"XDG_CONFIG_HOME="+filepath.Join(nodeHome, ".config"),)- cmdNode := exec.Command(nodeBin, "run", "--control-plane", lb.URL,- "--listen", "/ip4/127.0.0.1/tcp/0",+ samNode := launchNode(t, nodeBin, env, nodeHome, "run", "--control-plane", lb.URL,"--jwt-path", jwtPath,- "--bind-addr", "127.0.0.1:0","--api-token-path", tokenPath(t, "dummy-token"),"--allow-loopback","--monitor-bootstrap", "1s",@@ -214,37 +213,11 @@ roles: []"--autorelay-backoff", "1s","--autorelay-boot-delay", "0s",)- cmdNode.Dir = repoRoot(t)- cmdNode.Env = env- var stdoutNode, stderrNode safeBuffer- cmdNode.Stdout = &stdoutNode- cmdNode.Stderr = &stderrNode+ samNode.waitForAPI(t)+ nodePeerID := samNode.peerID.String()- if err := cmdNode.Start(); err != nil {- t.Fatal(err)- }- defer func() { _ = cmdNode.Process.Kill(); _ = cmdNode.Wait() }()-- // Wait for Node to get a reservation on Router A- var nodePeerID string- var out string- for i := 0; i < 100; i++ {- out = stdoutNode.String() + stderrNode.String()- if strings.Contains(out, "PeerID:") {- idx := strings.Index(out, "PeerID:")- parts := strings.Split(strings.TrimSpace(out[idx+len("PeerID:"):]), "\n")- if len(parts) > 0 {- nodePeerID = strings.TrimSpace(parts[0])- }- }- if nodePeerID != "" && strings.Contains(out, "Yielding static relays to AutoRelay") {- break- }- time.Sleep(100 * time.Millisecond)- }- if nodePeerID == "" {- t.Fatalf("Node failed to get PeerID in time.\nOutput:\n%s", out)- }+ // The node is on the mesh once Router A reports it connected to CP A.+ waitForPeerOnRouter(t, httpPortCP_A, testAdminToken, nodePeerID, 15*time.Second)t.Logf("Node started. Node PeerID: %s", nodePeerID)// Now FAILOVER: Switch LB to CP B and KILL CP A and Router A@@ -255,18 +228,9 @@ roles: []_ = cmdCP_A.Process.Kill()_ = cmdRouterA.Process.Kill()- // Wait for Node's AutoRelay to get updated- for i := 0; i < 150; i++ {- out = stdoutNode.String() + stderrNode.String()- if strings.Contains(out, "Successfully reconnected to router via HTTP fallback") {- break- }- time.Sleep(100 * time.Millisecond)- }- if !strings.Contains(out, "Successfully reconnected to router via HTTP fallback") {- t.Fatalf("Node failed to detect failover and reconnect.\nOutput:\n%s", out)- }- t.Log("Node successfully reconnected to router B!")+ // The node has failed over once Router B reports it connected to CP B.+ waitForPeerOnRouter(t, httpPortCP_B, testAdminToken, nodePeerID, 20*time.Second)+ t.Log("Node reconnected to router B")// Final verification: Ensure we can actually reach Node B via the Router B relayrelayAddrStr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s/p2p-circuit/p2p/%s", routerPortB, peerIDB, nodePeerID)@@ -311,7 +275,7 @@ roles: []}if connectErr != nil {- t.Fatalf("Failed to connect to Node B via router B relay: %v\nOutput: %s", connectErr, stdoutNode.String()+stderrNode.String())+ t.Fatalf("Failed to connect to Node B via router B relay: %v\n--- node.log ---\n%s", connectErr, samNode.log())}t.Log("Successfully connected to Node B via router B relay circuit!")diff --git a/tests/integration/fallback_test.go b/tests/integration/fallback_test.goindex f8ffb9cf..704e49c7 100644--- a/tests/integration/fallback_test.go+++ b/tests/integration/fallback_test.go@@ -16,31 +16,15 @@ package integration_testimport ("bytes"- "crypto/ed25519"- "crypto/rand"- "encoding/json""fmt"- "io"-- "os"- "os/exec""path/filepath"- "strings"+ "slices""sync"- "sync/atomic""testing""time"- "net/http"- "net/http/httptest"-"github.com/biscuit-auth/biscuit-go/v2/parser"- "github.com/google/sam/api"- "github.com/libp2p/go-libp2p"- "github.com/libp2p/go-libp2p/core/host"- "github.com/libp2p/go-libp2p/core/network"- "github.com/libp2p/go-msgio"- "google.golang.org/protobuf/proto"+ "github.com/google/sam/internal/node")// init forces the biscuit-go parser to build its underlying participle@@ -68,230 +52,92 @@ func (s *safeBuffer) String() string {}// TestSelfHealingHTTPFallback: the routers a node stored can be gone by its-// next start (a redeploy moves every router's address); the node must ask the-// control plane for the current ones and reach one. The assertions are the two-// ends of that path: the control plane saw the request, and the router that-// only exists since the address change completed an auth handshake with the-// node. Neither depends on what the node logs.+// next start (a redeploy moves every router's address); the node must ask+// the control plane for the current ones, reach one, and remember it.func TestSelfHealingHTTPFallback(t *testing.T) {nodeBin := buildBinary(t, "./cmd/sam-node")-- var mu sync.Mutex- var currentP2PAddr string- var infoRequests atomic.Int32-- pub, priv, err := ed25519.GenerateKey(rand.Reader)- if err != nil {- t.Fatalf("Failed to generate control plane key: %v", err)+ tmpDir := t.TempDir()+ oidcURL, mintToken := startCustomMockOIDC(t)++ // Leases are short so the control plane forgets the old router soon+ // after it is gone, the way a redeploy replaces the router set.+ meshDir := filepath.Join(tmpDir, "mesh")+ cpPort, _ := startControlPlane(t, meshDir, oidcURL, meshPolicyFile(t, meshDir), "--lease-duration", "2s")+ cpURL := fmt.Sprintf("http://127.0.0.1:%d", cpPort)+ oldRouter, stopOldRouter := startRouter(t, meshDir, cpPort, mintToken, "old-router")++ home := filepath.Join(tmpDir, "home")+ env, dataDir := nodeHome(home)+ token := tokenPath(t, "node-token")++ // Enroll while the old router is the only one, so it is the one stored.+ first := launchNode(t, nodeBin, env, home, "run", "--control-plane", cpURL,+ "--jwt", mintToken(map[string]interface{}{"sub": jwtUser}), "--allow-loopback", "--api-token-path", token)+ first.waitForAPI(t)+ waitForPeerOnRouter(t, cpPort, testAdminToken, first.peerID.String(), 10*time.Second)+ first.kill()+ if addrs := storedRouters(t, dataDir); !slices.Contains(addrs, oldRouter) {+ t.Fatalf("stored routers %v do not include the router the node enrolled through, %s", addrs, oldRouter)}- // createNewHost is a router as the node sees it: the auth handshake, and a- // channel closed once a node has completed it.- createNewHost := func() (host.Host, <-chan struct{}) {- newH, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))- if err != nil {- t.Fatal(err)- }- authenticated := make(chan struct{})- var once sync.Once- newH.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {- defer func() { _ = s.Close() }()- reader := msgio.NewVarintReaderSize(s, 1024*64)- msg, err := reader.ReadMsg()- if err != nil {- return- }- defer reader.ReleaseMsg(msg)-- writer := msgio.NewVarintWriter(s)- resp := &api.AuthResponse{- Success: true,- Biscuit: createMockBiscuitToken(t, newH.ID().String(), priv, api.RoleRouter, nil),- }- respBytes, err := proto.Marshal(resp)- if err != nil {- t.Errorf("marshal auth response: %v", err)- return- }- if err := writer.WriteMsg(respBytes); err != nil {- t.Errorf("write auth response: %v", err)- return- }- once.Do(func() { close(authenticated) })- })- return newH, authenticated+ // The redeploy: the stored router is gone, a new one is up, and only the+ // control plane knows about it.+ stopOldRouter()+ newRouter, _ := startRouter(t, meshDir, cpPort, mintToken, "new-router")+ waitForRouterSet(t, cpPort, []string{newRouter}, 10*time.Second)++ // The node comes back on its stored config and no --control-plane: what+ // it stored is a dead address, so reaching the new router at all means+ // it asked the control plane. /healthz answers only once Start succeeded,+ // and Start fails unless a router authenticated the node.+ again := launchNode(t, nodeBin, env, home, "run", "--allow-loopback", "--api-token-path", token)+ again.waitForAPI(t)+ lease := waitForPeerOnRouter(t, cpPort, testAdminToken, again.peerID.String(), 10*time.Second)+ if !slices.Contains(lease.Addresses, newRouter) {+ t.Fatalf("the node is connected to %v, not to the new router %s", lease.Addresses, newRouter)}- h, _ := createNewHost()- defer func() { _ = h.Close() }()-- mu.Lock()- currentP2PAddr = h.Addrs()[0].String() + "/p2p/" + h.ID().String()- mu.Unlock()-- mux := http.NewServeMux()-- // Mock OIDC server for device flow- mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- _ = json.NewEncoder(w).Encode(map[string]interface{}{- "issuer": "http://" + r.Host,- "token_endpoint": "http://" + r.Host + "/token",- "authorization_endpoint": "http://" + r.Host + "/auth",- })- })- mux.HandleFunc("/device/code", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- _ = json.NewEncoder(w).Encode(map[string]interface{}{- "device_code": "dev_code_123",- "user_code": "ABCD-1234",- "verification_uri": "http://example.com/verify",- "verification_uri_complete": "http://example.com/verify?code=ABCD-1234",- "expires_in": 60,- "interval": 1,- })- })- mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- _ = json.NewEncoder(w).Encode(map[string]string{- "access_token": "test-jwt-token",- "id_token": "test-jwt-token",- })- })-- mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {- mu.Lock()- addr := currentP2PAddr- mu.Unlock()+ // Healed for good: the next start does not need the control plane to+ // find a router either.+ again.kill()+ addrs := storedRouters(t, dataDir)+ if !slices.Contains(addrs, newRouter) || slices.Contains(addrs, oldRouter) {+ t.Fatalf("stored routers %v: want %s and not %s", addrs, newRouter, oldRouter)+ }+}- body, err := io.ReadAll(r.Body)- if err != nil {- http.Error(w, "Failed to read body", http.StatusBadRequest)- return- }- var enrollReq api.EnrollRequest- if err := proto.Unmarshal(body, &enrollReq); err != nil {- http.Error(w, "Invalid request", http.StatusBadRequest)- return- }+// storedRouters is the router set the node would dial on its next start.+func storedRouters(t *testing.T, dataDir string) []string {+ t.Helper()+ store, err := node.NewStore(dataDir)+ if err != nil {+ t.Fatalf("open node store: %v", err)+ }+ defer func() { _ = store.Close() }()+ _, addrs, err := store.LoadMeshConfig()+ if err != nil {+ t.Fatalf("load mesh config: %v", err)+ }+ return addrs+}- resp := &api.EnrollResponse{- BiscuitToken: createMockBiscuitToken(t, enrollReq.PeerId, priv, api.RoleNode, nil),- ControlPlanePublicKey: pub,- RouterAddresses: []string{addr},- }- data, err := proto.Marshal(resp)- if err != nil {- t.Errorf("marshal /register: %v", err)- http.Error(w, err.Error(), http.StatusInternalServerError)- return+// waitForRouterSet waits until the control plane's active routers listen on+// exactly want.+func waitForRouterSet(t *testing.T, cpPort int, want []string, timeout time.Duration) {+ t.Helper()+ deadline := time.Now().Add(timeout)+ for {+ var got []string+ for _, lease := range fetchAdminStatus(t, cpPort, testAdminToken).ActiveRouters {+ got = append(got, lease.Addresses...)}- w.Header().Set("Content-Type", "application/x-protobuf")- if _, err := w.Write(data); err != nil {- t.Errorf("write /register: %v", err)- }- })- mux.HandleFunc("/info", func(w http.ResponseWriter, r *http.Request) {- infoRequests.Add(1)- mu.Lock()- addr := currentP2PAddr- mu.Unlock()- resp := &api.ControlPlaneInfoResponse{- OidcIssuer: "http://" + r.Host,- ClientId: "sam-mesh-audience",- Audience: "sam-mesh-audience",- RouterAddresses: []string{addr},- }- data, err := proto.Marshal(resp)- if err != nil {- t.Errorf("marshal /info: %v", err)- http.Error(w, err.Error(), http.StatusInternalServerError)+ slices.Sort(got)+ if slices.Equal(got, want) {return}- w.Header().Set("Content-Type", "application/x-protobuf")- if _, err := w.Write(data); err != nil {- t.Errorf("write /info: %v", err)+ if time.Now().After(deadline) {+ t.Fatalf("control plane :%d routers = %v, want %v", cpPort, got, want)}- })-- httpServer := httptest.NewServer(mux)- defer httpServer.Close()-- tmpHome := t.TempDir()- env := append(os.Environ(),- "HOME="+tmpHome,- "XDG_CONFIG_HOME="+filepath.Join(tmpHome, ".config"),- "BROWSER=echo",- )-- // Step 1: Enroll via Join (using mock OIDC)- joinStdout, joinStderr, err := runCommandWithCallback(- t,- repoRoot(t),- 5*time.Second,- env,- "",- nodeBin,- "join",- httpServer.URL,- )- if err != nil {- t.Fatalf("Join failed: %v\nstdout:\n%s\nstderr:\n%s", err, joinStdout, joinStderr)- }-- out := joinStdout + joinStderr- if !strings.Contains(out, "Successfully joined the Sovereign Agent Mesh!") {- t.Fatalf("Join did not succeed:\n%s", out)- }-- // Step 2: Simulate router changing its P2P port (HTTP URL stays the same).- // The stored address now points at nothing; only /info knows the new one.- _ = h.Close()- newRouter, authenticated := createNewHost()- defer func() { _ = newRouter.Close() }()-- mu.Lock()- currentP2PAddr = newRouter.Addrs()[0].String() + "/p2p/" + newRouter.ID().String()- mu.Unlock()- infoRequests.Store(0)-- // Step 3: Start sam-node run, with everything it needs to stay up: a- // node that reaches the router and then exits has not healed.- runCmd := exec.Command(nodeBin, "run",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", fmt.Sprintf("127.0.0.1:%d", getFreePort(t)),- "--api-token-path", tokenPath(t, "fallback-test-token"),- )- runCmd.Env = env- var output safeBuffer- runCmd.Stdout = &output- runCmd.Stderr = &output-- if err := runCmd.Start(); err != nil {- t.Fatal(err)- }- exited := make(chan error, 1)- go func() { exited <- runCmd.Wait() }()- defer func() {- _ = runCmd.Process.Kill()- <-exited- }()-- select {- case <-authenticated:- case err := <-exited:- t.Fatalf("sam-node run exited (%v) before authenticating with the moved router.\nOutput:\n%s", err, output.String())- case <-time.After(5 * time.Second):- t.Fatalf("sam-node run never authenticated with the moved router.\nOutput:\n%s", output.String())- }- if infoRequests.Load() == 0 {- t.Fatal("the node reached the moved router without asking the control plane for its address")- }-- // Reaching the router is not the end: the node must stay up on it.- select {- case err := <-exited:- t.Fatalf("sam-node run exited (%v) right after authenticating.\nOutput:\n%s", err, output.String())- case <-time.After(500 * time.Millisecond):+ time.Sleep(200 * time.Millisecond)}}diff --git a/tests/integration/login_test.go b/tests/integration/login_test.goindex de1da1d4..28b7c5b1 100644--- a/tests/integration/login_test.go+++ b/tests/integration/login_test.go@@ -15,109 +15,142 @@package integration_testimport (- "context"+ "bytes""encoding/json"- "net/http"- "net/http/httptest"+ "fmt""os""path/filepath""strings""testing""time"+ "github.com/google/sam/api""github.com/google/sam/internal/node")-func TestSamNodeJoin(t *testing.T) {- nodeBin := buildBinary(t, "./cmd/sam-node")- tmpHome := t.TempDir()- env := append(os.Environ(),- "HOME="+tmpHome,- "XDG_CONFIG_HOME="+filepath.Join(tmpHome, ".config"),- "BROWSER=echo",- )+// These tests cover how a node comes to hold, keep and lose its mesh+// identity. They watch the two places that identity lives: the control+// plane's enrollment records and the node's own store. What the node prints+// along the way is not asserted on.- // Mock OIDC server for device flow- mux := http.NewServeMux()- mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- if err := json.NewEncoder(w).Encode(map[string]interface{}{- "issuer": "http://" + r.Host,- "token_endpoint": "http://" + r.Host + "/token",- "authorization_endpoint": "http://" + r.Host + "/auth",- }); err != nil {- t.Errorf("Failed to encode response: %v", err)- }- })- mux.HandleFunc("/device/code", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- if err := json.NewEncoder(w).Encode(map[string]interface{}{- "device_code": "dev_code_123",- "user_code": "ABCD-1234",- "verification_uri": "http://example.com/verify",- "verification_uri_complete": "http://example.com/verify?code=ABCD-1234",- "expires_in": 60,- "interval": 1,- }); err != nil {- t.Errorf("Failed to encode response: %v", err)- }- })- mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- if err := json.NewEncoder(w).Encode(map[string]string{- "access_token": "test-jwt-token",- "id_token": "test-jwt-token",- }); err != nil {- t.Errorf("Failed to encode response: %v", err)- }- })- oidcServer := httptest.NewServer(mux)- defer oidcServer.Close()-- // Start mock libp2p router that knows about our mock OIDC server- _, routerAddr := startMockRouterWithOIDC(t, oidcServer.URL)-- stdout, stderr, err := runCommandWithCallback(- t,- repoRoot(t),- 5*time.Second,- env,- "", // No stdin needed- nodeBin,- "join",- routerAddr,- )+const testAdminToken = "test-admin-token"++// jwtUser is the subject of tokens handed to a node directly with --jwt.+const jwtUser = "mock-user"++// meshPolicyFile writes the policy these tests run under: the mock+// provider's browser user and jwtUser are seated as nodes.+func meshPolicyFile(t *testing.T, dir string) string {+ t.Helper()+ if err := os.MkdirAll(dir, 0o755); err != nil {+ t.Fatal(err)+ }+ policy := fmt.Sprintf("bindings:\n - role: %s\n members: [%q, %q]\nroles: []\n",+ api.RoleNode, "user:"+mockOIDCUser, "user:"+jwtUser)+ policyFile := filepath.Join(dir, "policies.yaml")+ if err := os.WriteFile(policyFile, []byte(policy), 0o644); err != nil {+ t.Fatal(err)+ }+ return policyFile+}++// startMesh starts a control plane and a router under dir, with the policy+// from meshPolicyFile. It returns the control plane's port and URL.+func startMesh(t *testing.T, dir, oidcURL string, mintToken func(map[string]interface{}) string) (int, string) {+ t.Helper()+ cpPort, cleanup := startControlPlaneAndRouter(t, dir, oidcURL, mintToken, meshPolicyFile(t, dir))+ t.Cleanup(cleanup)+ return cpPort, fmt.Sprintf("http://127.0.0.1:%d", cpPort)+}++// nodeHome is the environment of a node whose files live under home, and+// the store that environment resolves to.+func nodeHome(home string) ([]string, string) {+ env := append(os.Environ(), "HOME="+home, "XDG_CONFIG_HOME="+filepath.Join(home, ".config"))+ return env, nodeDataDir(env, nil)+}++// storedMembership is what the node keeps of its enrollment: the biscuit+// and the control plane that issued it, both empty when there is none. The+// store is the node's while it runs, so this is for a node that has stopped.+func storedMembership(t *testing.T, dataDir string) ([]byte, string) {+ t.Helper()+ store, err := node.NewStore(dataDir)if err != nil {- t.Fatalf("join command failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)+ t.Fatalf("open node store: %v", err)}+ defer func() { _ = store.Close() }()+ identity, _ := store.LoadIdentity()+ cpURL, err := store.LoadControlPlaneURL()+ if err != nil {+ t.Fatalf("load control plane URL: %v", err)+ }+ return identity, cpURL+}- out := stdout + stderr- if !strings.Contains(out, "Successfully joined the Sovereign Agent Mesh!") {- t.Fatalf("join did not succeed:\n%s", out)+// join runs `sam-node join` for the node living under env, with the mock+// provider playing the user who signs in.+func join(t *testing.T, nodeBin string, env []string, cpURL string, extra ...string) {+ t.Helper()+ args := append([]string{"join"}, extra...)+ args = append(args, cpURL)+ stdout, stderr, err := runCommandWithCallback(t, repoRoot(t), 15*time.Second, env, "", nodeBin, args...)+ if err != nil {+ t.Fatalf("sam-node %s: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, stdout, stderr)}+}- // Verify that the identity is stored and node can run- stdout, stderr, err = runCommand(- t,- repoRoot(t),- 3*time.Second,- env,- "",- nodeBin,- "run", "--control-plane", routerAddr,- "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",- "--api-token-path", tokenPath(t, "dummy-token"),- )- if err != context.DeadlineExceeded {- t.Fatalf("expected run command to keep running, got: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)+// enrolledAs fails unless the control plane holds an enrollment for peerID+// made by subject, and returns it.+func enrolledAs(t *testing.T, cpPort int, peerID string, subject string) []byte {+ t.Helper()+ record := fetchAdminStatus(t, cpPort, testAdminToken).enrolledNode(peerID)+ if record == nil {+ t.Fatalf("control plane :%d has no enrollment for %s", cpPort, peerID)+ }+ if record.Role != api.RoleNode {+ t.Fatalf("enrollment role = %q, want %q", record.Role, api.RoleNode)}+ var claims struct {+ Sub string `json:"sub"`+ }+ if err := json.Unmarshal([]byte(record.ClaimsJSON), &claims); err != nil {+ t.Fatalf("enrollment claims %q: %v", record.ClaimsJSON, err)+ }+ if claims.Sub != subject {+ t.Fatalf("enrolled by %q, want %q", claims.Sub, subject)+ }+ return record.Biscuit+}++func TestSamNodeJoin(t *testing.T) {+ nodeBin := buildBinary(t, "./cmd/sam-node")+ tmpDir := t.TempDir()+ oidcURL, mintToken := startCustomMockOIDC(t)+ cpPort, cpURL := startMesh(t, filepath.Join(tmpDir, "mesh"), oidcURL, mintToken)++ home := filepath.Join(tmpDir, "home")+ env, dataDir := nodeHome(home)+ peerID := ensureNodeKey(t, dataDir).String()- out = stdout + stderr- if !strings.Contains(out, "Using stored identity.") {- t.Fatalf("node did not use stored identity:\n%s", out)+ join(t, nodeBin, env, cpURL)++ // The control plane enrolled this node for the user who signed in, and+ // the node holds the identity it was issued for that mesh.+ issued := enrolledAs(t, cpPort, peerID, mockOIDCUser)+ identity, storedURL := storedMembership(t, dataDir)+ if !bytes.Equal(identity, issued) {+ t.Fatal("the node's stored identity is not the biscuit the control plane issued")+ }+ if storedURL != cpURL {+ t.Fatalf("stored control plane = %q, want %q", storedURL, cpURL)}++ // From here on the node needs neither a login nor a --control-plane: it+ // comes up on what it stored and the router admits it.+ n := launchNode(t, nodeBin, env, home, "run", "--allow-loopback", "--api-token-path", tokenPath(t, "node-token"))+ n.waitForAPI(t)+ waitForPeerOnRouter(t, cpPort, testAdminToken, peerID, 10*time.Second)}// TestSamNodeJoinOfflineAccessSavesRefreshToken guards against a real bug:@@ -126,199 +159,161 @@ func TestSamNodeJoin(t *testing.T) {// breaking --offline-access for both "join" and "run --join".func TestSamNodeJoinOfflineAccessSavesRefreshToken(t *testing.T) {nodeBin := buildBinary(t, "./cmd/sam-node")- tmpHome := t.TempDir()- env := append(os.Environ(),- "HOME="+tmpHome,- "XDG_CONFIG_HOME="+filepath.Join(tmpHome, ".config"),- "BROWSER=echo",- )-- mux := http.NewServeMux()- mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- if err := json.NewEncoder(w).Encode(map[string]interface{}{- "issuer": "http://" + r.Host,- "token_endpoint": "http://" + r.Host + "/token",- "authorization_endpoint": "http://" + r.Host + "/auth",- }); err != nil {- t.Errorf("Failed to encode response: %v", err)- }- })- mux.HandleFunc("/device/code", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- if err := json.NewEncoder(w).Encode(map[string]interface{}{- "device_code": "dev_code_123",- "user_code": "ABCD-1234",- "verification_uri": "http://example.com/verify",- "verification_uri_complete": "http://example.com/verify?code=ABCD-1234",- "expires_in": 60,- "interval": 1,- }); err != nil {- t.Errorf("Failed to encode response: %v", err)- }- })- mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")- if err := json.NewEncoder(w).Encode(map[string]string{- "access_token": "test-jwt-token",- "id_token": "test-jwt-token",- "refresh_token": "test-refresh-token",- }); err != nil {- t.Errorf("Failed to encode response: %v", err)- }- })- oidcServer := httptest.NewServer(mux)- defer oidcServer.Close()+ tmpDir := t.TempDir()+ oidcURL, mintToken := startCustomMockOIDC(t)+ cpPort, cpURL := startMesh(t, filepath.Join(tmpDir, "mesh"), oidcURL, mintToken)- _, routerAddr := startMockRouterWithOIDC(t, oidcServer.URL)+ home := filepath.Join(tmpDir, "home")+ env, dataDir := nodeHome(home)+ peerID := ensureNodeKey(t, dataDir).String()- stdout, stderr, err := runCommandWithCallback(- t, repoRoot(t), 5*time.Second, env, "",- nodeBin, "join", "--offline-access", routerAddr,- )- if err != nil {- t.Fatalf("join command failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)- }- if !strings.Contains(stdout+stderr, "Successfully joined the Sovereign Agent Mesh!") {- t.Fatalf("join did not succeed:\n%s", stdout+stderr)- }+ join(t, nodeBin, env, cpURL, "--offline-access")+ enrolledAs(t, cpPort, peerID, mockOIDCUser)- store, err := node.NewStore(filepath.Join(tmpHome, ".config", "sam-mesh"))+ store, err := node.NewStore(dataDir)if err != nil {- t.Fatalf("failed to open store: %v", err)+ t.Fatalf("open node store: %v", err)}defer func() { _ = store.Close() }()-- refreshToken, err := store.LoadRefreshToken()+ if _, err := store.LoadRefreshToken(); err != nil {+ t.Fatalf("no refresh token saved after join --offline-access: %v", err)+ }+ issuer, clientID, _, err := store.LoadOIDCConfig()if err != nil {- t.Fatalf("expected a refresh token to be saved after --offline-access join, got error: %v", err)+ t.Fatal(err)}- if refreshToken != "test-refresh-token" {- t.Errorf("expected saved refresh token %q, got %q", "test-refresh-token", refreshToken)+ if issuer != oidcURL || clientID == "" {+ t.Fatalf("stored OIDC config (%q, %q) is not what the control plane advertised (%q)", issuer, clientID, oidcURL)}}-// TestSamNodeRunJoinNonInteractive covers the safe fallback: since the test-// harness gives the child process no TTY (stdin defaults to /dev/null),-// "run --join" must not attempt (and block on) an interactive OIDC login; it-// should fall back to the same unauthenticated MCP sidecar as plain "run".-func TestSamNodeRunJoinNonInteractive(t *testing.T) {+// TestSamNodeRunWithoutIdentityServesEnrollment covers a node that has never+// enrolled: it must not fail, nor block on a login it cannot complete+// without a terminal, but serve the enrollment-only MCP surface until+// someone enrolls it.+func TestSamNodeRunWithoutIdentityServesEnrollment(t *testing.T) {nodeBin := buildBinary(t, "./cmd/sam-node")- tmpHome := t.TempDir()- env := []string{- "HOME=" + tmpHome,- "XDG_CONFIG_HOME=" + filepath.Join(tmpHome, ".config"),+ for _, tc := range []struct {+ name string+ args []string+ }{+ {"plain run", nil},+ // --join without a TTY has nobody to log in; the node falls back+ // rather than hanging on a prompt. The control plane is never+ // contacted, so any address will do.+ {"run --join without a TTY", []string{"--join", "--control-plane", fmt.Sprintf("http://127.0.0.1:%d", getFreePort(t))}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ home := t.TempDir()+ env, dataDir := nodeHome(home)+ n := launchNode(t, nodeBin, env, home, append([]string{"run"}, tc.args...)...)+ n.waitForEnrollmentSidecar(t)+ n.staysUp(t, time.Second)+ n.kill()+ if identity, _ := storedMembership(t, dataDir); len(identity) != 0 {+ t.Fatal("a node that never enrolled holds an identity")+ }+ })}+}++// TestSamNodeRunWithStoredIdentity covers the everyday restart: a node that+// enrolled once comes back with no token and no --control-plane, on what it+// stored, and the router admits it again.+func TestSamNodeRunWithStoredIdentity(t *testing.T) {+ nodeBin := buildBinary(t, "./cmd/sam-node")+ tmpDir := t.TempDir()+ oidcURL, mintToken := startCustomMockOIDC(t)+ cpPort, cpURL := startMesh(t, filepath.Join(tmpDir, "mesh"), oidcURL, mintToken)++ home := filepath.Join(tmpDir, "home")+ env, dataDir := nodeHome(home)+ token := tokenPath(t, "node-token")- _, routerAddr := startMockRouter(t)-- stdout, stderr, err := runCommand(- t,- repoRoot(t),- 3*time.Second,- env,- "", // no stdin: child sees no TTY, same as a container without -it- nodeBin,- "run", "--join", "--control-plane", routerAddr,- "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",+ first := launchNode(t, nodeBin, env, home, "run",+ "--control-plane", cpURL,+ "--jwt", mintToken(map[string]interface{}{"sub": jwtUser}),+ "--allow-loopback",+ "--api-token-path", token,)- if err != context.DeadlineExceeded {- t.Fatalf("expected run --join to keep running via the unauthenticated sidecar, got: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)- }+ first.waitForAPI(t)+ issued := enrolledAs(t, cpPort, first.peerID.String(), jwtUser)+ first.kill()- out := stdout + stderr- if !strings.Contains(out, "no interactive terminal available") {- t.Fatalf("expected --join to warn about falling back without a TTY:\n%s", out)+ // /healthz answers only once Start succeeded, and Start fails unless a+ // router authenticated the node: coming up is the proof.+ again := launchNode(t, nodeBin, env, home, "run", "--allow-loopback", "--api-token-path", token)+ again.waitForAPI(t)+ if again.peerID != first.peerID {+ t.Fatalf("peer ID changed across restarts: %s then %s", first.peerID, again.peerID)}- if !strings.Contains(out, "Starting unauthenticated sidecar for enrollment over MCP") {- t.Fatalf("expected the unauthenticated sidecar to start:\n%s", out)+ again.kill()+ identity, _ := storedMembership(t, dataDir)+ if !bytes.Equal(identity, issued) {+ t.Fatal("restart replaced the stored identity instead of using it")}}-// TestSamNodeRunControlPlaneMismatch covers a node enrolled with one control-// plane being pointed at a different one: it must fail loudly (never-// silently keep using the originally-enrolled mesh), with or without-// --join, and "reset" must clear the way for a fresh enrollment elsewhere.+// TestSamNodeRunControlPlaneMismatch covers a node enrolled with one mesh+// being pointed at another: it must refuse, with or without --join, keeping+// its identity for the first mesh and leaving no trace on the second, and+// "reset" must clear the way for a fresh enrollment there.func TestSamNodeRunControlPlaneMismatch(t *testing.T) {nodeBin := buildBinary(t, "./cmd/sam-node")- tmpHome := t.TempDir()- env := []string{- "HOME=" + tmpHome,- "XDG_CONFIG_HOME=" + filepath.Join(tmpHome, ".config"),- }+ tmpDir := t.TempDir()+ oidcURL, mintToken := startCustomMockOIDC(t)+ cpPortA, cpURLA := startMesh(t, filepath.Join(tmpDir, "meshA"), oidcURL, mintToken)+ cpPortB, cpURLB := startMesh(t, filepath.Join(tmpDir, "meshB"), oidcURL, mintToken)- _, routerA := startMockRouter(t)- _, routerB := startMockRouter(t)-- // Enroll against control plane A.- stdout, stderr, err := runCommand(- t, repoRoot(t), 3*time.Second, env, "",- nodeBin, "run", "--control-plane", routerA, "--jwt", "test-jwt",- "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",- "--api-token-path", tokenPath(t, "dummy-token"),- )- if err != context.DeadlineExceeded {- t.Fatalf("expected initial enroll+run against A to keep running, got: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)- }+ home := filepath.Join(tmpDir, "home")+ env, dataDir := nodeHome(home)+ token := tokenPath(t, "node-token")+ jwt := mintToken(map[string]interface{}{"sub": jwtUser})- // Pointing the same store at B, without --join, must fail rather than- // silently keep talking to A.- stdout, stderr, err = runCommand(- t, repoRoot(t), 5*time.Second, env, "",- nodeBin, "run", "--control-plane", routerB,- "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",- "--api-token-path", tokenPath(t, "dummy-token"),- )- if err == nil || err == context.DeadlineExceeded {- t.Fatalf("expected run against a mismatched control plane to fail fast, got: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)- }- out := stdout + stderr- if !strings.Contains(out, "does not match the mesh") {- t.Fatalf("expected a control-plane mismatch error, got:\n%s", out)- }+ // Enroll with mesh A.+ onA := launchNode(t, nodeBin, env, home, "run", "--control-plane", cpURLA, "--jwt", jwt, "--allow-loopback", "--api-token-path", token)+ onA.waitForAPI(t)+ peerID := onA.peerID.String()+ issuedByA := enrolledAs(t, cpPortA, peerID, jwtUser)+ onA.kill()- // Same mismatch with --join, but no TTY to confirm the switch: must also- // fail fast rather than silently ignore --join or block indefinitely.- stdout, stderr, err = runCommand(- t, repoRoot(t), 5*time.Second, env, "",- nodeBin, "run", "--join", "--control-plane", routerB,- "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",- "--api-token-path", tokenPath(t, "dummy-token"),- )- if err == nil || err == context.DeadlineExceeded {- t.Fatalf("expected run --join against a mismatched control plane to fail fast without a TTY, got: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)- }- out = stdout + stderr- if !strings.Contains(out, "does not match the mesh") {- t.Fatalf("expected a control-plane mismatch error, got:\n%s", out)+ // Pointed at B, with or without --join (no TTY to confirm a switch),+ // the node refuses rather than reusing A's identity or enrolling anew.+ for _, args := range [][]string{+ {"run", "--control-plane", cpURLB},+ {"run", "--join", "--control-plane", cpURLB},+ } {+ n := launchNode(t, nodeBin, env, home, append(args, "--allow-loopback", "--api-token-path", token)...)+ if err := n.exitsWithin(t, 10*time.Second); err == nil {+ t.Fatalf("sam-node %s exited cleanly against a mesh it is not enrolled with", strings.Join(args, " "))+ }+ identity, storedURL := storedMembership(t, dataDir)+ if !bytes.Equal(identity, issuedByA) || storedURL != cpURLA {+ t.Fatalf("sam-node %s changed the stored membership (control plane now %q)", strings.Join(args, " "), storedURL)+ }+ if fetchAdminStatus(t, cpPortB, testAdminToken).enrolledNode(peerID) != nil {+ t.Fatalf("sam-node %s enrolled with mesh B", strings.Join(args, " "))+ }}- // "reset" clears the way for a fresh enrollment against B.- stdout, stderr, err = runCommand(t, repoRoot(t), 3*time.Second, env, "", nodeBin, "reset")+ // "reset" forgets the membership and nothing else: the key, hence the+ // peer ID, stays.+ stdout, stderr, err := runCommand(t, repoRoot(t), 5*time.Second, env, "", nodeBin, "reset")if err != nil {- t.Fatalf("reset failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)+ t.Fatalf("sam-node reset: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)}- if !strings.Contains(stdout+stderr, "Cleared stored mesh identity") {- t.Fatalf("expected reset confirmation, got:\n%s", stdout+stderr)+ if identity, storedURL := storedMembership(t, dataDir); len(identity) != 0 || storedURL != "" {+ t.Fatalf("reset left a membership behind (control plane %q)", storedURL)}- stdout, stderr, err = runCommand(- t, repoRoot(t), 3*time.Second, env, "",- nodeBin, "run", "--control-plane", routerB, "--jwt", "test-jwt",- "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",- "--api-token-path", tokenPath(t, "dummy-token"),- )- if err != context.DeadlineExceeded {- t.Fatalf("expected re-enroll against B after reset to keep running, got: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr)+ onB := launchNode(t, nodeBin, env, home, "run", "--control-plane", cpURLB, "--jwt", jwt, "--allow-loopback", "--api-token-path", token)+ onB.waitForAPI(t)+ if onB.peerID.String() != peerID {+ t.Fatalf("reset changed the peer ID: %s then %s", peerID, onB.peerID)+ }+ enrolledAs(t, cpPortB, peerID, jwtUser)+ onB.kill()+ if _, storedURL := storedMembership(t, dataDir); storedURL != cpURLB {+ t.Fatalf("stored control plane = %q, want %q", storedURL, cpURLB)}}diff --git a/tests/integration/minimal_helpers_test.go b/tests/integration/minimal_helpers_test.goindex 75cf7ec4..5ed1a560 100644--- a/tests/integration/minimal_helpers_test.go+++ b/tests/integration/minimal_helpers_test.go@@ -25,6 +25,7 @@ import ("path/filepath""runtime""strings"+ "sync""testing""time"@@ -36,10 +37,10 @@ import ("net""net/http""net/http/httptest"- "net/url""github.com/biscuit-auth/biscuit-go/v2""github.com/google/sam/api"+ "github.com/google/sam/internal/storage""github.com/libp2p/go-libp2p"dht "github.com/libp2p/go-libp2p-kad-dht""github.com/libp2p/go-libp2p/core/network"@@ -57,16 +58,50 @@ func repoRoot(t *testing.T) string {return filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", ".."))}+func TestMain(m *testing.M) {+ code := m.Run()+ builtBinaries.Range(func(_, entry any) bool {+ if b := entry.(*binaryBuild); b.path != "" {+ _ = os.RemoveAll(filepath.Dir(b.path))+ }+ return true+ })+ os.Exit(code)+}++// builtBinaries holds one build per package for the whole test process:+// Go caches compilation, but every `go build -o` links again, and fifty+// tests each linking three binaries is a minute of nothing.+var builtBinaries sync.Map // pkgPath -> *binaryBuild++type binaryBuild struct {+ once sync.Once+ path string+ err error+}+func buildBinary(t *testing.T, pkgPath string) string {t.Helper()- root := repoRoot(t)- out := filepath.Join(t.TempDir(), filepath.Base(pkgPath))- cmd := exec.Command("go", "build", "-o", out, ".")- cmd.Dir = filepath.Join(root, pkgPath)- if output, err := cmd.CombinedOutput(); err != nil {- t.Fatalf("building %s failed: %v\n%s", pkgPath, err, string(output))- }- return out+ entry, _ := builtBinaries.LoadOrStore(pkgPath, &binaryBuild{})+ b := entry.(*binaryBuild)+ b.once.Do(func() {+ root := repoRoot(t)+ dir, err := os.MkdirTemp("", "sam-integration-bin-")+ if err != nil {+ b.err = err+ return+ }+ b.path = filepath.Join(dir, filepath.Base(pkgPath))+ cmd := exec.Command("go", "build", "-o", b.path, ".")+ cmd.Dir = filepath.Join(root, pkgPath)+ if output, err := cmd.CombinedOutput(); err != nil {+ b.err = fmt.Errorf("%v\n%s", err, output)+ }+ })+ if b.err != nil {+ t.Fatalf("building %s failed: %v", pkgPath, b.err)+ }+ return b.path}func runCommand(@@ -133,23 +168,18 @@ func (iw *interceptorWriter) Write(p []byte) (n int, err error) {if strings.Contains(s, "redirect_uri=") && strings.Contains(s, "state=") {iw.handled = truego func() {- // Parse URL from the buffer to extract state and redirect_uri- // The URL is printed like: " http://...redirect_uri=...&state=..."- lines := strings.Split(s, "\n")- for _, line := range lines {- if strings.Contains(line, "redirect_uri=") {- parts := strings.Split(strings.TrimSpace(line), " ")- for _, p := range parts {- if strings.HasPrefix(p, "http") {- u, err := url.Parse(p)- if err == nil {- redirectURI := u.Query().Get("redirect_uri")- state := u.Query().Get("state")- if redirectURI != "" && state != "" {- time.Sleep(100 * time.Millisecond)- _, _ = http.Get(redirectURI + "?code=dev_code_123&state=" + state)- }- }+ // Play the user: open the authorization URL the CLI printed. The+ // provider signs the user in and redirects to the CLI's loopback+ // callback, which the client follows.+ for _, line := range strings.Split(s, "\n") {+ if !strings.Contains(line, "redirect_uri=") {+ continue+ }+ for _, p := range strings.Split(strings.TrimSpace(line), " ") {+ if strings.HasPrefix(p, "http") {+ time.Sleep(100 * time.Millisecond)+ if resp, err := http.Get(p); err == nil {+ _ = resp.Body.Close()}}}@@ -296,106 +326,6 @@ func startMockRouterWithControlPlaneKey(t *testing.T) (peer.ID, string, ed25519.return h.ID(), httpServer.URL, pub}-func startMockRouterWithOIDC(t *testing.T, oidcIssuerURL string) (peer.ID, string) {- t.Helper()-- pub, priv, err := ed25519.GenerateKey(rand.Reader)- if err != nil {- t.Fatalf("Failed to generate control plane key: %v", err)- }-- h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))- if err != nil {- t.Fatalf("failed to create mock libp2p host: %v", err)- }-- h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {- defer func() { _ = s.Close() }()- reader := msgio.NewVarintReaderSize(s, 1024*64)- msg, err := reader.ReadMsg()- if err != nil {- return- }- defer reader.ReleaseMsg(msg)-- writer := msgio.NewVarintWriter(s)- resp := &api.AuthResponse{- Success: true,- Biscuit: createMockBiscuitToken(t, h.ID().String(), priv, api.RoleRouter, nil),- }- respBytes, _ := proto.Marshal(resp)- _ = writer.WriteMsg(respBytes)- })-- kdht, err := dht.New(h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/sam"))- if err != nil {- t.Fatalf("failed to create DHT on mock router: %v", err)- }-- // Start HTTP server for enrollment and info-- mux := http.NewServeMux()- mux.HandleFunc("/info", func(w http.ResponseWriter, r *http.Request) {- if r.Method != http.MethodGet {- http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)- return- }- resp := &api.ControlPlaneInfoResponse{- OidcIssuer: oidcIssuerURL,- ClientId: "sam-mesh-audience",- Audience: "sam-mesh-audience",- }- data, err := proto.Marshal(resp)- if err != nil {- http.Error(w, "Failed to marshal response", http.StatusInternalServerError)- return- }- w.Header().Set("Content-Type", "application/x-protobuf")- w.WriteHeader(http.StatusOK)- _, _ = w.Write(data)- })- mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {- if r.Method != http.MethodPost {- http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)- return- }- body, err := io.ReadAll(r.Body)- if err != nil {- http.Error(w, "Failed to read body", http.StatusBadRequest)- return- }- var req api.EnrollRequest- if err := proto.Unmarshal(body, &req); err != nil {- http.Error(w, "Invalid request format", http.StatusBadRequest)- return- }-- resp := &api.EnrollResponse{- BiscuitToken: createMockBiscuitToken(t, req.PeerId, priv, api.RoleNode, req.Labels),- ControlPlanePublicKey: pub,- RouterAddresses: []string{h.Addrs()[0].String() + "/p2p/" + h.ID().String()},- }- data, err := proto.Marshal(resp)- if err != nil {- http.Error(w, "Failed to marshal response", http.StatusInternalServerError)- return- }- w.Header().Set("Content-Type", "application/x-protobuf")- w.WriteHeader(http.StatusOK)- _, _ = w.Write(data)- })-- httpServer := httptest.NewServer(mux)-- t.Cleanup(func() {- httpServer.Close()- _ = kdht.Close()- _ = h.Close()- })-- return h.ID(), httpServer.URL-}-func createMockBiscuitToken(t *testing.T, peerID string, priv ed25519.PrivateKey, role string, labels map[string]string) []byte {builder := biscuit.NewBuilder(priv)err := builder.AddAuthorityFact(biscuit.Fact{Predicate: biscuit.Predicate{Name: "target_unrestricted"}})@@ -472,11 +402,27 @@ func createMockBiscuitToken(t *testing.T, peerID string, priv ed25519.PrivateKey}func startControlPlaneAndRouter(t *testing.T, tmpDir string, oidcURL string, mintToken func(map[string]interface{}) string, policyFile string) (int, func()) {- cpBin := buildBinary(t, "./cmd/sam-control-plane")- routerBin := buildBinary(t, "./cmd/sam-router")+ cpPort, stopCP := startControlPlane(t, tmpDir, oidcURL, policyFile)+ _, stopRouter := startRouter(t, tmpDir, cpPort, mintToken, "router")++ // Wait for router lease to be active and registered+ fetchPeerID(t, cpPort)++ cleanup := func() {+ stopRouter()+ stopCP()+ }++ return cpPort, cleanup+}+// startControlPlane starts a sam-control-plane under tmpDir, trusting oidcURL+// and holding the policy in policyFile with the router role added, plus any+// extra flags. It returns the port and a stop function.+func startControlPlane(t *testing.T, tmpDir string, oidcURL string, policyFile string, extra ...string) (int, func()) {+ t.Helper()+ cpBin := buildBinary(t, "./cmd/sam-control-plane")cpPort := getFreePort(t)- routerPort := getFreePort(t)// Automatically adjust the policy file to grant the "router" role to group "routers"originalPolicy, err := os.ReadFile(policyFile)@@ -484,29 +430,40 @@ func startControlPlaneAndRouter(t *testing.T, tmpDir string, oidcURL string, minwritePolicyWithRouter(t, policyFile, string(originalPolicy))}- // 1. Start Control Plane- cpCmd := exec.Command(cpBin,+ cpCmd := exec.Command(cpBin, append([]string{"--bind-address", fmt.Sprintf("127.0.0.1:%d", cpPort),"--db-dsn", filepath.Join(tmpDir, "cp-keys.db"),"--issuer", oidcURL,"--insecure-skip-tls-verify","--admin-token-path", tokenPath(t, "test-admin-token"),- )+ }, extra...)...)cpCmd.Stdout = os.StdoutcpCmd.Stderr = os.Stderrif err := cpCmd.Start(); err != nil {t.Fatalf("failed to start control plane: %v", err)}+ stop := func() {+ _ = cpCmd.Process.Kill()+ _ = cpCmd.Wait()+ }+ t.Cleanup(stop)- // Wait for CP to be upwaitForControlPlane(t, cpPort)-- // Inject policy into CP database via APIinjectPolicyYAML(t, cpPort, "test-admin-token", policyFile)+ return cpPort, stop+}++// startRouter starts a sam-router named name against the control plane on+// cpPort, renewing its lease every second so the control plane's view of the+// router's peers is current. It returns the router's p2p address and a stop+// function.+func startRouter(t *testing.T, tmpDir string, cpPort int, mintToken func(map[string]interface{}) string, name string) (string, func()) {+ t.Helper()+ routerBin := buildBinary(t, "./cmd/sam-router")+ routerPort := getFreePort(t)- // 2. Start RouterrouterJWT := mintToken(map[string]interface{}{- "sub": "router-integration-1",+ "sub": name,"groups": []string{"routers"},"roles": []string{api.RoleRouter},})@@ -514,29 +471,40 @@ func startControlPlaneAndRouter(t *testing.T, tmpDir string, oidcURL string, minrouterCmd := exec.Command(routerBin,"--control-plane", fmt.Sprintf("http://127.0.0.1:%d", cpPort),"--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", routerPort),- "--keys-path", filepath.Join(tmpDir, "router-keys.db"),+ "--keys-path", filepath.Join(tmpDir, name+"-keys.db"),"--allow-loopback","--oidc-token", routerJWT,+ // Each renewal carries the router's connected peers, which is how+ // tests see a node reach the mesh through the control plane.+ "--lease-renew-interval", "1s",)routerCmd.Stdout = os.StdoutrouterCmd.Stderr = os.Stderrif err := routerCmd.Start(); err != nil {- _ = cpCmd.Process.Kill()- _ = cpCmd.Wait()t.Fatalf("failed to start router: %v", err)}-- // Wait for router lease to be active and registered- fetchPeerID(t, cpPort)-- cleanup := func() {+ stop := func() {_ = routerCmd.Process.Kill()_ = routerCmd.Wait()- _ = cpCmd.Process.Kill()- _ = cpCmd.Wait()}+ t.Cleanup(stop)- return cpPort, cleanup+ // The router's peer ID is its own; the control plane learns it from the+ // first lease, which is also when the router is ready for nodes.+ deadline := time.Now().Add(10 * time.Second)+ for {+ for _, lease := range fetchAdminStatus(t, cpPort, "test-admin-token").ActiveRouters {+ for _, addr := range lease.Addresses {+ if strings.HasPrefix(addr, fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/", routerPort)) {+ return addr, stop+ }+ }+ }+ if time.Now().After(deadline) {+ t.Fatalf("router %s never leased with control plane :%d", name, cpPort)+ }+ time.Sleep(100 * time.Millisecond)+ }}func getFreePort(t *testing.T) int {@@ -550,6 +518,74 @@ func getFreePort(t *testing.T) int {return port}+// adminStatus is the control plane's view of the mesh, in the types it+// serializes on /admin/status.+type adminStatus struct {+ EnrolledNodes []storage.EnrolledNode `json:"enrolled_nodes"`+ ActiveRouters []storage.RouterLease `json:"active_routers"`+}++func fetchAdminStatus(t *testing.T, cpPort int, adminToken string) adminStatus {+ t.Helper()+ req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/admin/status", cpPort), nil)+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Authorization", "Bearer "+adminToken)+ client := &http.Client{Timeout: 5 * time.Second}+ resp, err := client.Do(req)+ if err != nil {+ t.Fatalf("GET /admin/status: %v", err)+ }+ defer func() { _ = resp.Body.Close() }()+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("GET /admin/status: %s", resp.Status)+ }+ var status adminStatus+ if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {+ t.Fatalf("decode /admin/status: %v", err)+ }+ return status+}++// enrolledNode is the control plane's record of peerID, or nil.+func (s adminStatus) enrolledNode(peerID string) *storage.EnrolledNode {+ for i := range s.EnrolledNodes {+ if s.EnrolledNodes[i].PeerID == peerID {+ return &s.EnrolledNodes[i]+ }+ }+ return nil+}++// routerWith is the lease of a router that reports peerID connected, or nil.+func (s adminStatus) routerWith(peerID string) *storage.RouterLease {+ for i := range s.ActiveRouters {+ for _, p := range s.ActiveRouters[i].ConnectedPeers {+ if p == peerID {+ return &s.ActiveRouters[i]+ }+ }+ }+ return nil+}++// waitForPeerOnRouter returns the lease of the router peerID is connected+// to, as the control plane learns it from the router's lease renewals.+func waitForPeerOnRouter(t *testing.T, cpPort int, adminToken string, peerID string, timeout time.Duration) *storage.RouterLease {+ t.Helper()+ deadline := time.Now().Add(timeout)+ for {+ if lease := fetchAdminStatus(t, cpPort, adminToken).routerWith(peerID); lease != nil {+ return lease+ }+ if time.Now().After(deadline) {+ t.Fatalf("no router on control plane :%d reported %s connected within %v", cpPort, peerID, timeout)+ }+ time.Sleep(200 * time.Millisecond)+ }+}+func fetchPeerID(t *testing.T, port int) string {ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)defer cancel()diff --git a/tests/integration/mock_oidc_test.go b/tests/integration/mock_oidc_test.goindex 708e20a9..54f12c32 100644--- a/tests/integration/mock_oidc_test.go+++ b/tests/integration/mock_oidc_test.go@@ -17,17 +17,31 @@ package integration_testimport ("crypto/rand""crypto/rsa"+ "crypto/sha256""encoding/base64"+ "encoding/hex""encoding/json""math/big""net/http""net/http/httptest"+ "net/url"+ "strings"+ "sync""testing""time""github.com/golang-jwt/jwt/v5")+// mockOIDCUser is the subject the mock provider signs in as whoever+// completes its browser authorization flow.+const mockOIDCUser = "browser-user"++// startCustomMockOIDC is the identity provider the mesh trusts in these+// tests: discovery, a JWKS, and the authorization code grant with PKCE that+// an interactive join drives. The tokens it issues are real RS256 JWTs a+// control plane verifies against the JWKS. mintToken issues one directly,+// for components that are configured with a token instead of logging in.func startCustomMockOIDC(t *testing.T) (string, func(claims map[string]interface{}) string) {privKey, err := rsa.GenerateKey(rand.Reader, 2048)if err != nil {@@ -42,8 +56,10 @@ func startCustomMockOIDC(t *testing.T) (string, func(claims map[string]interfacemux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")_ = json.NewEncoder(w).Encode(map[string]interface{}{- "issuer": issuer,- "jwks_uri": issuer + "/keys",+ "issuer": issuer,+ "jwks_uri": issuer + "/keys",+ "authorization_endpoint": issuer + "/auth",+ "token_endpoint": issuer + "/token",})})@@ -81,5 +97,91 @@ func startCustomMockOIDC(t *testing.T) (string, func(claims map[string]interfacereturn jwtStr}+ // Authorization codes handed out by /auth, redeemed once at /token.+ type authCode struct {+ challenge string+ redirectURI string+ offline bool+ }+ var mu sync.Mutex+ codes := map[string]authCode{}+ newCode := func() string {+ b := make([]byte, 16)+ if _, err := rand.Read(b); err != nil {+ t.Fatalf("random code: %v", err)+ }+ return hex.EncodeToString(b)+ }++ // The user opens this URL; the provider signs them in as mockOIDCUser+ // and sends them back to the client with a code bound to its PKCE+ // challenge.+ mux.HandleFunc("/auth", func(w http.ResponseWriter, r *http.Request) {+ q := r.URL.Query()+ if q.Get("response_type") != "code" || q.Get("client_id") == "" || q.Get("redirect_uri") == "" || q.Get("code_challenge_method") != "S256" {+ http.Error(w, "unsupported authorization request", http.StatusBadRequest)+ return+ }+ code := newCode()+ mu.Lock()+ codes[code] = authCode{+ challenge: q.Get("code_challenge"),+ redirectURI: q.Get("redirect_uri"),+ offline: strings.Contains(q.Get("scope"), "offline_access"),+ }+ mu.Unlock()+ back, err := url.Parse(q.Get("redirect_uri"))+ if err != nil {+ http.Error(w, "bad redirect_uri", http.StatusBadRequest)+ return+ }+ bq := back.Query()+ bq.Set("code", code)+ bq.Set("state", q.Get("state"))+ back.RawQuery = bq.Encode()+ http.Redirect(w, r, back.String(), http.StatusFound)+ })++ oauthError := func(w http.ResponseWriter, code string) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusBadRequest)+ _ = json.NewEncoder(w).Encode(map[string]string{"error": code})+ }+ mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {+ if err := r.ParseForm(); err != nil {+ oauthError(w, "invalid_request")+ return+ }+ if r.PostForm.Get("grant_type") != "authorization_code" {+ oauthError(w, "unsupported_grant_type")+ return+ }+ mu.Lock()+ granted, ok := codes[r.PostForm.Get("code")]+ delete(codes, r.PostForm.Get("code"))+ mu.Unlock()+ verifier := sha256.Sum256([]byte(r.PostForm.Get("code_verifier")))+ if !ok || granted.redirectURI != r.PostForm.Get("redirect_uri") ||+ base64.RawURLEncoding.EncodeToString(verifier[:]) != granted.challenge {+ oauthError(w, "invalid_grant")+ return+ }+ idToken := mintToken(map[string]interface{}{+ "sub": mockOIDCUser,+ "email": mockOIDCUser + "@example.com",+ })+ resp := map[string]interface{}{+ "access_token": idToken,+ "id_token": idToken,+ "token_type": "Bearer",+ "expires_in": 3600,+ }+ if granted.offline {+ resp["refresh_token"] = newCode()+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(resp)+ })+return issuer, mintToken}diff --git a/tests/integration/node_helpers_test.go b/tests/integration/node_helpers_test.goindex c66c9622..58ebd2dc 100644--- a/tests/integration/node_helpers_test.go+++ b/tests/integration/node_helpers_test.go@@ -15,6 +15,7 @@package integration_testimport (+ "context""fmt""net/http""os"@@ -26,6 +27,7 @@ import ("github.com/google/sam/internal/node""github.com/libp2p/go-libp2p/core/peer"+ "github.com/modelcontextprotocol/go-sdk/mcp")// A node under test is observed through what it does, never through what it@@ -240,6 +242,58 @@ func (n *backgroundNode) staysUp(t *testing.T, d time.Duration) {}}+// waitForEnrollmentSidecar returns once the node serves the enrollment-only+// MCP surface: a session opened without a token that offers+// get_login_instructions. A node holding an identity refuses token-less+// sessions, so this also tells which of the two sidecars came up.+func (n *backgroundNode) waitForEnrollmentSidecar(t *testing.T) {+ t.Helper()+ deadline := time.Now().Add(10 * time.Second)+ var last error+ for time.Now().Before(deadline) {+ select {+ case err := <-n.exited:+ n.exited <- err+ t.Fatalf("sam-node exited (%v) before serving its enrollment sidecar.\n--- node.log ---\n%s", err, n.log())+ default:+ }+ tools, err := listMCPTools(n.apiAddr)+ if err == nil {+ for _, name := range tools {+ if name == "get_login_instructions" {+ return+ }+ }+ err = fmt.Errorf("tools %v do not include get_login_instructions", tools)+ }+ last = err+ time.Sleep(100 * time.Millisecond)+ }+ t.Fatalf("sam-node at %s did not serve the enrollment sidecar in time: %v\n--- node.log ---\n%s", n.apiAddr, last, n.log())+}++// listMCPTools opens a token-less MCP session against apiAddr and returns the+// names of the tools it offers.+func listMCPTools(apiAddr string) ([]string, error) {+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()+ client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "0.1.0"}, nil)+ session, err := client.Connect(ctx, &mcp.StreamableClientTransport{Endpoint: "http://" + apiAddr + "/mcp"}, nil)+ if err != nil {+ return nil, err+ }+ defer func() { _ = session.Close() }()+ res, err := session.ListTools(ctx, nil)+ if err != nil {+ return nil, err+ }+ names := make([]string, 0, len(res.Tools))+ for _, tool := range res.Tools {+ names = append(names, tool.Name)+ }+ return names, nil+}+// waitForAPI polls addr's /healthz for a node the test started itself.func waitForAPI(t *testing.T, addr string) {t.Helper()diff --git a/tests/integration/p2p_test.go b/tests/integration/p2p_test.godeleted file mode 100644index ac8a89a8..00000000--- a/tests/integration/p2p_test.go+++ /dev/null@@ -1,40 +0,0 @@-// Copyright 2026 Google LLC-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--package integration_test--import (- "os"- "path/filepath"- "strings"- "testing"- "time"-)--func TestSamNodeRunWithoutIdentity(t *testing.T) {- nodeBin := buildBinary(t, "./cmd/sam-node")- tmpHome := t.TempDir()- env := []string{- "HOME=" + tmpHome,- "XDG_CONFIG_HOME=" + filepath.Join(tmpHome, ".config"),- }- stdout, stderr, err := runCommand(t, repoRoot(t), 10*time.Second, append(os.Environ(), env...), "", nodeBin, "run")- if err == nil {- t.Fatalf("expected sam-node run without identity to fail, but it succeeded\nstdout:\n%s\nstderr:\n%s", stdout, stderr)- }- out := stdout + stderr- if !strings.Contains(out, "No identity found. Starting unauthenticated sidecar for enrollment over MCP") {- t.Fatalf("expected missing identity message, got:\n%s", out)- }-}diff --git a/tests/integration/proxy_test.go b/tests/integration/proxy_test.godeleted file mode 100644index 2ccf1d10..00000000--- a/tests/integration/proxy_test.go+++ /dev/null@@ -1,101 +0,0 @@-// Copyright 2026 Google LLC-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--package integration_test--import (- "context"- "crypto/ed25519"- "crypto/rand"- "encoding/json"- "os"- "path/filepath"- "strings"- "testing"- "time"-- "go.etcd.io/bbolt"-)--func TestSamNodeRunWithStoredIdentity(t *testing.T) {- nodeBin := buildBinary(t, "./cmd/sam-node")- tmpHome := t.TempDir()-- // Pre-populate store- configDir := filepath.Join(tmpHome, ".config", "sam-mesh")- err := os.MkdirAll(configDir, 0700)- if err != nil {- t.Fatalf("failed to create config dir: %v", err)- }-- dbPath := filepath.Join(configDir, "agent.db")- db, err := bbolt.Open(dbPath, 0600, nil)- if err != nil {- t.Fatalf("failed to open db: %v", err)- }- err = db.Update(func(tx *bbolt.Tx) error {- b, err := tx.CreateBucketIfNotExists([]byte("identity"))- if err != nil {- return err- }- pub, priv, err := ed25519.GenerateKey(rand.Reader)- if err != nil {- return err- }- biscuitBytes := createMockBiscuitToken(t, "12D3KooWRHDt3Ajd1t7YikBXyK2Uw1wrJwEX88XbwJhHh1XtBNkx", priv, "sam:role:node", nil)- if err := b.Put([]byte("identity_biscuit"), biscuitBytes); err != nil {- return err- }- if err := b.Put([]byte("control_plane_public_key"), pub); err != nil {- return err- }- addrsData, _ := json.Marshal([]string{"/ip4/127.0.0.1/tcp/4002/p2p/Qm..."})- if err := b.Put([]byte("router_addresses"), addrsData); err != nil {- return err- }- return nil- })- if err != nil {- t.Fatalf("failed to update db: %v", err)- }- _ = db.Close()-- env := append(os.Environ(), "HOME="+tmpHome)-- runOut, runErrOut, err := runCommand(- t,- repoRoot(t),- 3*time.Second,- env,- "",- nodeBin,- "run", "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",- "--listen", "/ip4/127.0.0.1/tcp/0",- "--bind-addr", "127.0.0.1:0",- "--api-token-path", tokenPath(t, "dummy-token"),- "--data-dir", configDir,- )- if err != context.DeadlineExceeded {- t.Fatalf("expected run command to keep running until timeout, got: %v\nstdout:\n%s\nstderr:\n%s", err, runOut, runErrOut)- }- t.Logf("stdout:\n%s\nstderr:\n%s", runOut, runErrOut)-- out := runOut + runErrOut- if !strings.Contains(out, "Using stored identity.") {- t.Fatalf("run command did not use stored identity:\n%s", out)- }- if !strings.Contains(out, "SAM Node Online") {- t.Fatalf("node did not reach online state:\n%s", out)- }-}