Skip to content

Agent Mesh

tests: an ephemeral fleet is forgotten in seconds

Antonio Ojeae11634d84816
diff --git a/tests/integration/node_lifecycle_test.go b/tests/integration/node_lifecycle_test.go
index b7f9afd7..7687c954 100644
--- a/tests/integration/node_lifecycle_test.go
+++ b/tests/integration/node_lifecycle_test.go
@@ -22,6 +22,7 @@ import (
"net/http"
"os"
"path/filepath"
+ "slices"
"strings"
"sync"
"testing"
@@ -91,6 +92,92 @@ func (m *oidcMember) refresh(t *testing.T, cpURL, jwt string) (int, string) {
return resp.StatusCode, string(data)
}
+// bootstrapMember is a member enrolled with a bootstrap token, driven over
+// the control plane API alone: the test decides when it refreshes.
+type bootstrapMember struct {
+ priv crypto.PrivKey
+ peerID peer.ID
+ biscuit []byte
+}
+
+// enrollBootstrapMember spends one use of token at POST /enroll.
+func enrollBootstrapMember(t *testing.T, cpURL, token string) *bootstrapMember {
+ t.Helper()
+ priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pID, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pub, err := crypto.MarshalPublicKey(priv.GetPublic())
+ if err != nil {
+ t.Fatal(err)
+ }
+ ts := time.Now().UnixMilli()
+ sig, err := priv.Sign(api.EnrollChallenge(pID.String(), ts))
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, err := proto.Marshal(&api.BootstrapEnrollRequest{
+ BootstrapToken: token, PeerId: pID.String(), PublicKey: pub, RequestedRole: api.RoleNode,
+ ChallengeUnixMs: ts, ChallengeSignature: sig,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp, err := http.Post(cpURL+"/enroll", "application/x-protobuf", bytes.NewReader(body))
+ if err != nil {
+ t.Fatalf("POST /enroll: %v", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ data, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("POST /enroll: %d %s", resp.StatusCode, data)
+ }
+ var enrolled api.BootstrapEnrollResponse
+ if err := proto.Unmarshal(data, &enrolled); err != nil {
+ t.Fatal(err)
+ }
+ if enrolled.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED || len(enrolled.BiscuitToken) == 0 {
+ t.Fatalf("POST /enroll answered %s: %s", enrolled.Status, enrolled.ErrorMessage)
+ }
+ return &bootstrapMember{priv: priv, peerID: pID, biscuit: enrolled.BiscuitToken}
+}
+
+// refresh is one POST /refresh with the member's current credential; on
+// success the member holds the new one. It returns the status and body.
+func (m *bootstrapMember) refresh(t *testing.T, cpURL string) (int, string) {
+ t.Helper()
+ ts := time.Now().UnixMilli()
+ sig, err := m.priv.Sign(api.RefreshChallenge(m.peerID.String(), ts))
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, err := proto.Marshal(&api.TokenRefreshRequest{ChallengeUnixMs: ts, ChallengeSignature: sig})
+ if err != nil {
+ t.Fatal(err)
+ }
+ req, _ := http.NewRequest(http.MethodPost, cpURL+"/refresh", bytes.NewReader(body))
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(m.biscuit))
+ req.Header.Set("Content-Type", "application/x-protobuf")
+ resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)
+ if err != nil {
+ t.Fatalf("POST /refresh: %v", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ data, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode == http.StatusOK {
+ var refreshed api.TokenRefreshResponse
+ if err := proto.Unmarshal(data, &refreshed); err != nil {
+ t.Fatal(err)
+ }
+ m.biscuit = refreshed.BiscuitToken
+ }
+ return resp.StatusCode, string(data)
+}
+
// listNodes is GET /user/nodes as the administrator, one page.
func listNodes(t *testing.T, cpURL, adminToken, query string) *api.NodeListResponse {
t.Helper()
@@ -270,8 +357,8 @@ func TestLapsedMembersAreCollected(t *testing.T) {
}
time.Sleep(200 * time.Millisecond)
}
- if v, ok := controlPlaneMetric(t, cpURL, "agentmesh_control_plane_nodes_deleted_total"); !ok || v != leaving {
- t.Errorf("nodes_deleted_total = %v (present %v), want %d", v, ok, leaving)
+ if v, ok := controlPlaneMetric(t, cpURL, `agentmesh_control_plane_nodes_deleted_total{reason="session"}`); !ok || v != leaving {
+ t.Errorf("nodes_deleted_total{reason=session} = %v (present %v), want %d", v, ok, leaving)
}
stopRefreshing()
if err := <-refreshed; err != nil {
@@ -284,3 +371,118 @@ func TestLapsedMembersAreCollected(t *testing.T) {
t.Fatalf("admin status after the sweep lists %d nodes, want 1", len(nodes))
}
}
+
+// TestSilentEphemeralMembersAreForgotten is the ladder's fleet and the
+// probe's identities in seconds. A bootstrap enrollment has no session, so
+// a member that leaves used to stay enrolled forever; it is now forgotten
+// once it has not refreshed for the retention, and a member an ephemeral
+// token enrolled, which the ladder and the probes mint, within the
+// ephemeral retention. A regular member that is silent for less than the
+// retention, and one that refreshes, stay.
+func TestSilentEphemeralMembersAreForgotten(t *testing.T) {
+ tmpDir := t.TempDir()
+ oidcURL, _ := startCustomMockOIDC(t)
+ policyFile := filepath.Join(tmpDir, "policies.yaml")
+ if err := os.WriteFile(policyFile, []byte("roles: []\nbindings: []\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ // Ephemeral members are kept for a second after their last refresh,
+ // regular ones for an hour; the GC sweeps every second for the shorter
+ // of the two to matter.
+ cpPort, _ := startControlPlane(t, tmpDir, oidcURL, policyFile,
+ "--auto-approve-enrollment",
+ "--ephemeral-node-retention", "1s",
+ "--node-retention", "10s",
+ )
+ cpURL := fmt.Sprintf("http://127.0.0.1:%d", cpPort)
+
+ const fleet = 12
+ ephemeralToken := mintBootstrapTokenWithUses(t, cpURL, testAdminToken, fleet, true)
+ regularToken := mintBootstrapTokenWithUses(t, cpURL, testAdminToken, 2, false)
+ silent := enrollBootstrapMember(t, cpURL, regularToken)
+ refreshing := enrollBootstrapMember(t, cpURL, regularToken)
+ members := make([]*bootstrapMember, fleet)
+ for i := range members {
+ members[i] = enrollBootstrapMember(t, cpURL, ephemeralToken)
+ }
+ enrolledAt := time.Now()
+ if got := len(fetchAdminStatus(t, cpPort, testAdminToken).GetEnrolledNodes()); got != fleet+2 {
+ t.Fatalf("admin status lists %d nodes right after enrollment, want %d", got, fleet+2)
+ }
+ for _, n := range fetchAdminStatus(t, cpPort, testAdminToken).GetEnrolledNodes() {
+ if n.GetEphemeral() != (n.GetPeerId() != silent.peerID.String() && n.GetPeerId() != refreshing.peerID.String()) {
+ t.Fatalf("node %s ephemeral=%v, want the flag on the fleet's members only", n.GetPeerId(), n.GetEphemeral())
+ }
+ }
+
+ // The fleet falls silent and is forgotten; the regular members stay,
+ // the one that refreshes with its row's last refresh moving.
+ if status, body := refreshing.refresh(t, cpURL); status != http.StatusOK {
+ t.Fatalf("refresh: %d %s", status, body)
+ }
+ deadline := time.Now().Add(8 * time.Second)
+ for {
+ nodes := fetchAdminStatus(t, cpPort, testAdminToken).GetEnrolledNodes()
+ if len(nodes) == 2 {
+ var left []string
+ for _, n := range nodes {
+ left = append(left, n.GetPeerId())
+ }
+ slices.Sort(left)
+ want := []string{silent.peerID.String(), refreshing.peerID.String()}
+ slices.Sort(want)
+ if !slices.Equal(left, want) {
+ t.Fatalf("after the sweep the status lists %v, want the two regular members %v", left, want)
+ }
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("%d nodes still listed %s after the fleet enrolled; want the fleet forgotten after a second of silence", len(nodes), time.Since(enrolledAt))
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+ if v, ok := controlPlaneMetric(t, cpURL, `agentmesh_control_plane_nodes_deleted_total{reason="ephemeral"}`); !ok || v != fleet {
+ t.Errorf("nodes_deleted_total{reason=ephemeral} = %v (present %v), want %d", v, ok, fleet)
+ }
+ if v, ok := controlPlaneMetric(t, cpURL, `agentmesh_control_plane_nodes_deleted_total{reason="silent"}`); !ok || v != 0 {
+ t.Errorf("nodes_deleted_total{reason=silent} = %v (present %v), want 0: the regular members are within their retention", v, ok)
+ }
+
+ // A forgotten member is a stranger: its credential still verifies, its
+ // row is gone, so it is told to enroll again. The regular ones refresh.
+ if status, body := members[0].refresh(t, cpURL); status != http.StatusUnauthorized {
+ t.Fatalf("refresh of a forgotten member: %d %s, want 401", status, strings.TrimSpace(body))
+ }
+ for _, m := range []*bootstrapMember{silent, refreshing} {
+ if status, body := m.refresh(t, cpURL); status != http.StatusOK {
+ t.Fatalf("refresh of a regular member after the sweep: %d %s", status, body)
+ }
+ }
+}
+
+// mintBootstrapTokenWithUses mints a node bootstrap token good for uses
+// enrollments, ephemeral or not.
+func mintBootstrapTokenWithUses(t *testing.T, cpURL, adminToken string, uses int, ephemeral bool) string {
+ t.Helper()
+ body, err := protojson.Marshal(&api.BootstrapTokenCreateRequest{Role: api.RoleNode, TtlHours: 1, MaxUsages: int32(uses), Description: "lifecycle", Ephemeral: ephemeral})
+ if err != nil {
+ t.Fatal(err)
+ }
+ req, _ := http.NewRequest(http.MethodPost, cpURL+"/admin/bootstrap-tokens", bytes.NewReader(body))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Bearer "+adminToken)
+ resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ data, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode != http.StatusCreated {
+ t.Fatalf("POST /admin/bootstrap-tokens: %d %s", resp.StatusCode, data)
+ }
+ var created api.BootstrapTokenCreateResponse
+ if err := protojson.Unmarshal(data, &created); err != nil {
+ t.Fatal(err)
+ }
+ return created.GetToken()
+}
diff --git a/tests/scale/member-journey.sh b/tests/scale/member-journey.sh
index fe7a62df..2c67d100 100755
--- a/tests/scale/member-journey.sh
+++ b/tests/scale/member-journey.sh
@@ -196,7 +196,7 @@ admin_api() {
mint_token() {
local uses=$(( COUNT + 2 * LATE + 5 ))
admin_api POST /admin/bootstrap-tokens -H 'Content-Type: application/json' \
- -d "{\"role\":\"mesh:role:node\",\"max_usages\":${uses},\"ttl_hours\":48,\"description\":\"member-journey ${COUNT}+${LATE} from $(hostname)\"}" \
+ -d "{\"role\":\"mesh:role:node\",\"max_usages\":${uses},\"ttl_hours\":48,\"ephemeral\":true,\"description\":\"member-journey ${COUNT}+${LATE} from $(hostname)\"}" \
> "$OUT/token.json" || fail "minting a bootstrap token failed"
python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["token"])' "$OUT/token.json" > "$OUT/bootstrap-token"
TOKEN_ID=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["id"])' "$OUT/token.json")
diff --git a/tests/scale/member-ladder.sh b/tests/scale/member-ladder.sh
index 2dd4c28e..3b13057f 100755
--- a/tests/scale/member-ladder.sh
+++ b/tests/scale/member-ladder.sh
@@ -176,7 +176,7 @@ summarize() {
# One token for the whole ladder.
uses=$(( TOTAL + TOTAL / 10 + 20 ))
admin_api POST /admin/bootstrap-tokens -H 'Content-Type: application/json' \
- -d "{\"role\":\"mesh:role:node\",\"max_usages\":${uses},\"ttl_hours\":48,\"description\":\"member-ladder ${TOTAL} from ${#ALL_MINIONS[@]} minions\"}" > "$OUT/token.json" || fail "minting the bootstrap token failed"
+ -d "{\"role\":\"mesh:role:node\",\"max_usages\":${uses},\"ttl_hours\":48,\"ephemeral\":true,\"description\":\"member-ladder ${TOTAL} from ${#ALL_MINIONS[@]} minions\"}" > "$OUT/token.json" || fail "minting the bootstrap token failed"
python3 -c 'import json,sys; t=json.load(open(sys.argv[1])); open(sys.argv[2],"w").write(t["token"])' "$OUT/token.json" "$OUT/bootstrap-token"
TOKEN_ID=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["id"])' "$OUT/token.json")
rm -f "$OUT/token.json"; chmod 600 "$OUT/bootstrap-token"