Skip to content

Router that virtually distributes inference across connected devices in the home.

Go
13.6 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"
)

func TestInboundInitialCannotPublishAfterTeardown(t *testing.T) {
	m := newTestManagerPort(t, 15201)
	peer := newTestManagerPort(t, 15202)
	inviteID := "inbound-after-teardown"
	sess := &pairingSession{
		inviteID: inviteID,
		role:     roleJoiner,
		peerPairing: &PairingInfo{
			NodeUUID:  peer.identity.NodeUUID,
			NodeID:    peer.identity.NodeID,
			Name:      peer.identity.Name,
			ClusterID: "cluster-1",
		},
	}
	m.putSession(sess)
	inv := &Invite{
		InviteID:     inviteID,
		FromNodeUUID: peer.identity.NodeUUID,
		ClusterID:    "cluster-1",
		State:        inviteStatePending,
		CreatedAt:    time.Now().UnixMilli(),
	}

	if err := m.teardownClusterLocal(); err != nil {
		t.Fatal(err)
	}
	m.onJoinerInitialComplete(inv, sess)

	if _, ok := m.getInvite(inviteID); ok {
		t.Fatal("stale inbound Initial completion republished an invite after teardown")
	}
}

func TestFailedTeardownBlocksTrustAndReadmission(t *testing.T) {
	m := newTestManagerPort(t, 15203)
	peer := newPinFixture(t, "peer")
	pinTrusted(t, m, peer.uuid, peer.cert, peer.fp)
	m.upsertMember(&ClusterNode{
		NodeUUID: peer.uuid, ID: "peer", ClusterID: "cluster-1",
		AdmissionEpoch: 1, State: stateMember,
	})

	pinPath := m.trust.pinPath(peer.uuid)
	if err := os.Remove(pinPath); err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir(pinPath, 0o700); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(pinPath, "block-delete"), []byte("x"), 0o600); err != nil {
		t.Fatal(err)
	}
	if err := m.teardownClusterLocal(); err == nil {
		t.Fatal("teardown unexpectedly completed despite the pin-store failure")
	}

	block, _ := pem.Decode([]byte(peer.cert))
	if block == nil {
		t.Fatal("decode peer certificate")
	}
	cert, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		t.Fatal(err)
	}
	req := httptest.NewRequest(http.MethodPost, rosterPath, nil)
	req.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}}
	if _, ok := m.verifyClientPin(req); ok {
		t.Error("failed teardown left the old peer authorized for mTLS")
	}
	if _, _, err := m.foundCluster("new cluster"); err == nil {
		t.Error("new admission committed while teardown.pending still existed")
	}
}

func TestRosterPinWithoutPersistedMemberIsPruned(t *testing.T) {
	dir := t.TempDir()
	m := testManagerAt(t, dir, 15204)
	activateTestCluster(t, m, "cluster-1")
	peer := newPinFixture(t, "roster-peer")
	pinTrusted(t, m, peer.uuid, peer.cert, peer.fp)

	restarted := testManagerAt(t, dir, 15205)
	if _, ok := restarted.trust.Get(peer.uuid); ok {
		t.Fatal("restart retained a roster pin whose matching member was never persisted")
	}
}

func TestMalformedInitialDoesNotLeakSession(t *testing.T) {
	m := testManagerAt(t, t.TempDir(), 15206)
	inviteID := "malformed-initial"
	m.admissionMu.Lock()
	before := m.admissionCounter
	m.admissionMu.Unlock()
	rr := httptest.NewRecorder()
	m.handlePairingInitial(rr, &pairingEnvelope{InviteID: inviteID, Phase: "initial"}, []byte("not-eap-noob"), "127.0.0.1")
	if _, ok := m.getSession(inviteID); ok {
		t.Fatal("malformed unauthenticated Initial request retained a live pairing session")
	}
	m.admissionMu.Lock()
	after := m.admissionCounter
	m.admissionMu.Unlock()
	if after != before {
		t.Fatalf("malformed request consumed durable admission epoch: before=%d after=%d", before, after)
	}
}

func TestPreInviteSessionsExpireAndAreBounded(t *testing.T) {
	m := testManagerAt(t, t.TempDir(), 15210)
	expired := &pairingSession{
		inviteID: "pre-invite-expired", role: roleJoiner,
		createdAt: time.Now().Add(-time.Hour).UnixMilli(),
	}
	m.putSession(expired)
	inviteTTLOverride = time.Minute
	t.Cleanup(func() { inviteTTLOverride = 0 })
	m.expirePendingInvites(time.Now())
	if _, ok := m.getSession(expired.inviteID); ok {
		t.Fatal("pre-invite session survived its TTL")
	}

	for i := 0; i < maxPreInviteSessions; i++ {
		m.putSession(&pairingSession{
			inviteID: "bounded-" + strconv.Itoa(i), role: roleJoiner,
			createdAt: time.Now().UnixMilli(),
		})
	}
	rr := httptest.NewRecorder()
	m.handlePairingInitial(rr, &pairingEnvelope{InviteID: "over-limit", Phase: "initial"}, []byte(`{"Type":1}`), "127.0.0.1")
	if rr.Code != http.StatusTooManyRequests {
		t.Fatalf("over-limit status = %d, want %d", rr.Code, http.StatusTooManyRequests)
	}
}

func TestDirectPairingRejectsRemovedAdmission(t *testing.T) {
	m := newTestManagerPort(t, 15207)
	peer := newPinFixture(t, "readmission")
	pinTrusted(t, m, peer.uuid, peer.cert, peer.fp)
	m.upsertMember(&ClusterNode{
		NodeUUID: peer.uuid, ID: "readmission", ClusterID: "cluster-1",
		AdmissionEpoch: 1, State: stateMember,
	})
	proof, err := m.newRemovalProof(peer.uuid, 1)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := m.putRemovalProof(proof); err != nil {
		t.Fatal(err)
	}
	if m.removalProofBlocksAdmission(peer.uuid, "other-cluster", 1) {
		t.Fatal("cluster-scoped removal proof blocked admission to another cluster")
	}
	if err := m.trust.Remove(peer.uuid); err != nil {
		t.Fatal(err)
	}
	m.removeMemberByUUID(peer.uuid)

	pi, cert := pairingInfoFor(t, peer)
	sess := putInviterSession(t, m, "readmission", "cluster-1")
	committed, err := m.commitPairing(sess, pi, cert, time.Now().UnixMilli())
	if err == nil || committed {
		t.Fatalf("removed admission commit = %v, err = %v; want rejection", committed, err)
	}
}

func TestRestartRollsBackProvisionalAdmission(t *testing.T) {
	dir := t.TempDir()
	m := testManagerAt(t, dir, 15211)
	epoch, err := m.reserveAdmissionEpoch()
	if err != nil {
		t.Fatal(err)
	}
	peer := newPinFixture(t, "provisional-peer")
	pinTrusted(t, m, peer.uuid, peer.cert, peer.fp)
	m.upsertMember(&ClusterNode{
		NodeUUID: peer.uuid, ID: "provisional-peer", ClusterID: "cluster-new",
		AdmissionEpoch: 1, State: stateMember,
	})
	m.addSelfMemberForAdmission("cluster-new", epoch)
	if err := m.persistMembersErr(); err != nil {
		t.Fatal(err)
	}

	restarted := testManagerAt(t, dir, 15212)
	if cid, _ := restarted.currentAdmission(); cid != "" {
		t.Fatalf("provisional admission became active after restart: %q", cid)
	}
	if len(restarted.trust.List()) != 0 || len(restarted.snapshotNodes()) != 0 {
		t.Fatal("restart retained provisional members or pins")
	}
}

func TestBareRejectionRemovesDepartedPeerButKeepsCluster(t *testing.T) {
	m := newTestManagerPort(t, 15208)
	startPeerStub(t, m, "departed", http.StatusForbidden, false)

	m.reconcilePeersAndMaybeSelfRemove()

	if cid, _ := m.clusterIdentity(); cid != "cluster-1" {
		t.Fatalf("surviving cluster id = %q, want cluster-1", cid)
	}
	if nodes := m.snapshotNodes(); len(nodes) != 0 {
		t.Fatalf("departed peer remained in roster: %+v", nodes)
	}
}

func TestRemovalIgnoresUnrelatedCompositionChange(t *testing.T) {
	m := newTestManagerPort(t, 15209)
	target := newPinFixture(t, "target")
	pinTrusted(t, m, target.uuid, target.cert, target.fp)
	m.upsertMember(&ClusterNode{
		NodeUUID: target.uuid, ID: "target", ClusterID: "cluster-1",
		AdmissionEpoch: 1, State: stateMember,
	})
	targetUUID := target.uuid
	m.testRemovalPrepared = make(chan struct{})
	m.testRemovalContinue = make(chan struct{})
	done := make(chan struct{})
	go func() {
		m.handleNodesRemove(&Message{Params: mustJSON(t, removeParams{NodeUUID: &targetUUID})})
		close(done)
	}()
	<-m.testRemovalPrepared
	m.upsertMember(&ClusterNode{
		NodeUUID: "unrelated", ID: "unrelated", ClusterID: "cluster-1",
		AdmissionEpoch: 1, State: stateMember,
	})
	close(m.testRemovalContinue)
	<-done
	if _, ok := m.trust.Get(target.uuid); ok {
		t.Fatal("target pin survived because an unrelated member changed")
	}
	if _, ok := m.memberByNodeID(target.uuid); ok {
		t.Fatal("target member survived because an unrelated member changed")
	}
}

func TestRosterFixpointContinuesAfterAdmissionUpgrade(t *testing.T) {
	m := newTestManagerPort(t, 15213)
	aUUID, aCert, aFP, aPriv := makeNode(t, "node-a")
	pinTrusted(t, m, aUUID, aCert, aFP)
	m.upsertMember(&ClusterNode{
		NodeUUID: aUUID, ID: "node-a", ClusterID: "cluster-1",
		AdmissionEpoch: 1, State: stateMember,
	})
	cUUID, cCert, cFP, _ := makeNode(t, "node-c")
	_, selfEpoch := m.currentAdmission()
	endA2 := signEndorsement(m.identity.Signer, m.identity.NodeUUID,
		aUUID, aFP, "cluster-1", time.Now().UnixMilli(), 2, selfEpoch)
	endC := signEndorsement(aPriv, aUUID,
		cUUID, cFP, "cluster-1", time.Now().UnixMilli(), 1, 2)

	entries := []RosterEntry{
		{
			NodeUUID: cUUID, NodeID: "node-c", AdmissionEpoch: 1,
			CertPem: cCert, CertFingerprint: cFP, Endorsements: []Endorsement{endC},
		},
		{
			NodeUUID: aUUID, NodeID: "node-a", AdmissionEpoch: 2,
			CertPem: aCert, CertFingerprint: aFP, Endorsements: []Endorsement{endA2},
		},
	}
	if !m.applyMembers(entries, "cluster-1", aUUID) {
		t.Fatal("roster upgrade reported no change")
	}
	if _, ok := m.trust.Get(cUUID); !ok {
		t.Fatal("fixpoint stopped after upgrading the endorser admission")
	}
}

func TestJoinerFailureAfterInviterCommitRollsBackPairing(t *testing.T) {
	m := newTestManagerPort(t, 15214)
	peer := newPinFixture(t, "joiner")
	pi, cert := pairingInfoFor(t, peer)
	inviteID := "late-joiner-failure"
	m.putInvite(&Invite{
		InviteID: inviteID, ClusterID: "cluster-1",
		State: inviteStatePending, CreatedAt: time.Now().UnixMilli(),
	})
	sess := putInviterSession(t, m, inviteID, "cluster-1")
	if err := m.finalizePairing(inviteID, sess, pi, cert); err != nil {
		t.Fatal(err)
	}

	m.handlePairingFailed(httptest.NewRecorder(), &pairingEnvelope{
		InviteID: inviteID, Phase: "fail",
	})
	if _, ok := m.trust.Get(peer.uuid); !ok {
		t.Fatal("unauthenticated post-success failure rolled the inviter back")
	}
	m.handlePairingFailedFrom(httptest.NewRecorder(), &pairingEnvelope{
		InviteID: inviteID, Phase: "fail",
	}, peer.uuid)

	if _, ok := m.trust.Get(peer.uuid); ok {
		t.Fatal("inviter retained peer after joiner reported post-success commit failure")
	}
	if _, ok := m.memberByNodeID(peer.uuid); ok {
		t.Fatal("inviter retained member after joiner reported post-success commit failure")
	}
	invite, ok := m.getInvite(inviteID)
	if !ok || invite.State != inviteStateFailed {
		t.Fatalf("invite state = %+v, want failed", invite)
	}
}

func TestJoinerAckFinalizesInviterSession(t *testing.T) {
	m := newTestManagerPort(t, 15215)
	peer := newPinFixture(t, "joiner")
	pi, cert := pairingInfoFor(t, peer)
	inviteID := "joiner-ack"
	m.putInvite(&Invite{
		InviteID: inviteID, ClusterID: "cluster-1",
		State: inviteStatePending, CreatedAt: time.Now().UnixMilli(),
	})
	sess := putInviterSession(t, m, inviteID, "cluster-1")
	if err := m.finalizePairing(inviteID, sess, pi, cert); err != nil {
		t.Fatal(err)
	}

	m.handlePairingAck(httptest.NewRecorder(), &pairingEnvelope{
		InviteID: inviteID, Phase: "ack",
	}, "wrong-peer")
	if _, ok := m.getSession(inviteID); !ok {
		t.Fatal("unauthenticated acknowledgment finalized the inviter session")
	}
	m.handlePairingAck(httptest.NewRecorder(), &pairingEnvelope{
		InviteID: inviteID, Phase: "ack",
	}, peer.uuid)

	if _, ok := m.getSession(inviteID); ok {
		t.Fatal("acknowledged inviter session was not deleted")
	}
	if _, ok := m.trust.Get(peer.uuid); !ok {
		t.Fatal("acknowledgment removed the committed peer")
	}
	invite, ok := m.getInvite(inviteID)
	if !ok || invite.State != inviteStatePaired {
		t.Fatalf("invite state = %+v, want paired", invite)
	}
}

func TestFinalCompletionRetriesLostResponse(t *testing.T) {
	var attempts atomic.Int32
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		if attempts.Add(1) == 1 {
			conn, _, err := w.(http.Hijacker).Hijack()
			if err != nil {
				t.Error(err)
				return
			}
			_ = conn.Close()
			return
		}
		respondPairing(w, []byte("eap-success"))
	}))
	defer server.Close()

	blob, err := postCompletionBlob(server.Client(), server.Listener.Addr().String(),
		"retry-final", []byte(`{"Type":6}`))
	if err != nil {
		t.Fatal(err)
	}
	if string(blob) != "eap-success" || attempts.Load() != 2 {
		t.Fatalf("retry result = %q after %d attempts", blob, attempts.Load())
	}
}

func TestDuplicateFinalCompletionReturnsCachedSuccess(t *testing.T) {
	m := newTestManagerPort(t, 15216)
	inviteID := "cached-final"
	m.putInvite(&Invite{InviteID: inviteID, State: inviteStatePaired})
	m.putSession(&pairingSession{
		inviteID: inviteID, role: roleInviter, awaitingAck: true,
		completionResponse: []byte("cached-eap-success"),
	})
	rec := httptest.NewRecorder()
	m.handlePairingCompletion(rec, &pairingEnvelope{InviteID: inviteID}, []byte(`{"Type":6}`), "127.0.0.1")
	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "Y2FjaGVkLWVhcC1zdWNjZXNz") {
		t.Fatalf("cached completion response = %d %q", rec.Code, rec.Body.String())
	}
}

func TestAckTimeoutKeepsCommittedPairAndClearsInviteProvenance(t *testing.T) {
	m := newTestManagerPort(t, 15217)
	peer := newPinFixture(t, "joiner")
	pi, cert := pairingInfoFor(t, peer)
	inviteID := "ack-timeout"
	m.putInvite(&Invite{
		InviteID: inviteID, ClusterID: "cluster-1",
		State: inviteStatePending, CreatedAt: time.Now().UnixMilli(),
	})
	sess := putInviterSession(t, m, inviteID, "cluster-1")
	if err := m.finalizePairing(inviteID, sess, pi, cert); err != nil {
		t.Fatal(err)
	}
	m.setInviteCreatedCluster(true)
	sess.createdAt = time.Now().Add(-time.Hour).UnixMilli()
	inviteTTLOverride = time.Minute
	t.Cleanup(func() { inviteTTLOverride = 0 })
	m.expirePendingInvites(time.Now())

	if _, ok := m.getSession(inviteID); ok {
		t.Fatal("unacknowledged retry session survived its TTL")
	}
	if _, ok := m.trust.Get(peer.uuid); !ok {
		t.Fatal("ack timeout removed a fully committed peer")
	}
	if m.isInviteCreatedCluster() {
		t.Fatal("ack timeout left a real paired cluster marked invite-created")
	}
}