Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

16.4 KB
Raw Blame
package relay

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"errors"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/gorilla/websocket"
	"github.com/rs/zerolog"
)

type managedRelayServer struct {
	cmd     *exec.Cmd
	logBuf  *safeBuffer
	addr    string
	dataDir string
}

type safeBuffer struct {
	mu  sync.Mutex
	buf bytes.Buffer
}

func (b *safeBuffer) Write(p []byte) (int, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.buf.Write(p)
}

func (b *safeBuffer) String() string {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.buf.String()
}

func TestManagedRuntimeRelayRegistrationReconnectDrain(t *testing.T) {
	pulseRoot, pulseProRelayDir := managedRelayWorkspaceRoots(t)
	relayBinary := buildManagedRelayBinary(t, pulseProRelayDir)
	const revocationFeedToken = "managed-runtime-revocation-feed-token"
	revocationFeed := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/v1/revocations" {
			http.NotFound(w, r)
			return
		}
		if r.Header.Get("Authorization") != "Bearer "+revocationFeedToken {
			http.Error(w, "unauthorized", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(map[string]any{
			"next_seq": 0,
			"has_more": false,
			"events":   []any{},
		})
	}))
	defer revocationFeed.Close()
	revocationCertificate, err := x509.ParseCertificate(revocationFeed.TLS.Certificates[0].Certificate[0])
	if err != nil {
		t.Fatalf("parse revocation feed certificate: %v", err)
	}
	revocationCAPath := filepath.Join(t.TempDir(), "revocation-feed-ca.pem")
	revocationCAPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: revocationCertificate.Raw})
	if err := os.WriteFile(revocationCAPath, revocationCAPEM, 0o600); err != nil {
		t.Fatalf("write revocation feed CA: %v", err)
	}
	t.Setenv("SSL_CERT_FILE", revocationCAPath)
	t.Setenv("PULSE_RELAY_LICENSE_SERVER_URL", revocationFeed.URL)
	t.Setenv("PULSE_RELAY_REVOCATION_FEED_TOKEN", revocationFeedToken)

	publicKey, privateKey, err := ed25519.GenerateKey(nil)
	if err != nil {
		t.Fatalf("generate relay signing keypair: %v", err)
	}
	publicKeyB64 := base64.StdEncoding.EncodeToString(publicKey)
	licenseToken := signManagedRelayJWT(t, privateKey, map[string]any{
		"lid":   "lic_managed_runtime_relay",
		"email": "[email protected]",
		"tier":  "pro",
		"iat":   time.Now().Unix(),
		"exp":   time.Now().Add(30 * 24 * time.Hour).Unix(),
	})

	apiSlowStarted := make(chan struct{}, 1)
	apiSlowCancelled := make(chan struct{}, 1)
	apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/api/status":
			w.Header().Set("Content-Type", "application/json")
			_ = json.NewEncoder(w).Encode(map[string]any{
				"endpoint": "status",
				"ok":       true,
			})
		case "/api/slow-resource":
			select {
			case apiSlowStarted <- struct{}{}:
			default:
			}
			select {
			case <-r.Context().Done():
				select {
				case apiSlowCancelled <- struct{}{}:
				default:
				}
				return
			case <-time.After(30 * time.Second):
				t.Error("slow relay runtime request completed instead of being cancelled during drain")
				return
			}
		default:
			http.NotFound(w, r)
		}
	}))
	defer apiServer.Close()

	relayAddr := allocateLoopbackAddress(t)
	server1 := startManagedRelayServer(t, relayBinary, relayAddr, t.TempDir(), publicKeyB64)
	defer server1.stopKill()

	logBuffer := &safeBuffer{}
	logger := zerolog.New(logBuffer).With().Timestamp().Logger()

	client := NewClient(
		Config{
			Enabled:        true,
			ServerURL:      "ws://" + relayAddr + "/ws/instance",
			InstanceSecret: "managed-runtime-instance-secret",
		},
		ClientDeps{
			LicenseTokenFunc: func() string { return licenseToken },
			TokenValidator:   func(token string) bool { return token == "managed-runtime-app-token" },
			LocalAddr:        strings.TrimPrefix(apiServer.URL, "http://"),
			ServerVersion:    "6.0.0-rc.2",
			IdentityPubKey:   "managed-runtime-identity-pub",
		},
		logger,
	)

	runCtx, cancelRun := context.WithCancel(context.Background())
	defer cancelRun()
	runErrCh := make(chan error, 1)
	go func() {
		runErrCh <- client.Run(runCtx)
	}()
	defer func() {
		cancelRun()
		select {
		case err := <-runErrCh:
			if err != nil && !errors.Is(err, context.Canceled) {
				t.Fatalf("relay client run exited unexpectedly: %v\nclient logs:\n%s", err, logBuffer.String())
			}
		case <-time.After(5 * time.Second):
			t.Fatalf("timed out waiting for relay client shutdown\nclient logs:\n%s", logBuffer.String())
		}
	}()

	instanceID := waitForManagedRelayConnected(t, client, 20*time.Second)
	if !strings.HasPrefix(instanceID, "relay_") {
		t.Fatalf("managed runtime instance ID %q does not look canonical", instanceID)
	}
	verifyManagedRelayProxyRoundTrip(t, relayAddr, instanceID, "/api/status")

	// Phase 1: normal reconnect after abrupt relay restart with persistent data.
	server1.stopKill()
	server2 := startManagedRelayServer(t, relayBinary, relayAddr, server1.dataDir, publicKeyB64)
	defer server2.stopKill()
	waitForManagedRelayConnected(t, client, 20*time.Second)
	verifyManagedRelayProxyRoundTrip(t, relayAddr, instanceID, "/api/status")

	// Phase 2: stale-session recovery after relay restart with a fresh data dir.
	server2.stopKill()
	server3 := startManagedRelayServer(t, relayBinary, relayAddr, t.TempDir(), publicKeyB64)
	defer server3.stopKill()
	waitForManagedRelayLog(t, logBuffer, "relay session resume rejected, retrying fresh registration", 20*time.Second)
	waitForManagedRelayConnected(t, client, 20*time.Second)
	verifyManagedRelayProxyRoundTrip(t, relayAddr, instanceID, "/api/status")

	// Phase 3: drain during inflight work, then reconnect to a replacement server.
	appConn, channelID := openManagedRelayAppConnection(t, relayAddr, instanceID, "managed-runtime-app-token")
	sendManagedRelayProxyRequest(t, appConn, channelID, ProxyRequest{
		ID:     "req_drain_inflight",
		Method: "GET",
		Path:   "/api/slow-resource",
	})
	select {
	case <-apiSlowStarted:
	case <-time.After(5 * time.Second):
		t.Fatalf("timed out waiting for slow managed-runtime relay request to start\nclient logs:\n%s", logBuffer.String())
	}
	server3.stopTerm()
	server4 := startManagedRelayServer(t, relayBinary, relayAddr, server3.dataDir, publicKeyB64)
	defer server4.stopKill()
	waitForManagedRelayLog(t, logBuffer, "Relay server draining, will reconnect", 20*time.Second)
	select {
	case <-apiSlowCancelled:
	case <-time.After(5 * time.Second):
		t.Fatalf("timed out waiting for slow managed-runtime relay request cancellation\nclient logs:\n%s", logBuffer.String())
	}
	_ = appConn.Close()
	waitForManagedRelayConnected(t, client, 20*time.Second)
	verifyManagedRelayProxyRoundTrip(t, relayAddr, instanceID, "/api/status")
	waitForManagedRelayIdle(t, client, 5*time.Second)

	if !strings.Contains(logBuffer.String(), "relay session resume rejected, retrying fresh registration") {
		t.Fatalf("managed runtime exercise never observed stale-session recovery\nclient logs:\n%s", logBuffer.String())
	}
	if !strings.Contains(logBuffer.String(), "Relay server draining, will reconnect") {
		t.Fatalf("managed runtime exercise never observed drain handling\nclient logs:\n%s", logBuffer.String())
	}

	_ = pulseRoot // document intent: this test is intentionally workspace-scoped.
}

func managedRelayWorkspaceRoots(t *testing.T) (string, string) {
	t.Helper()
	_, currentFile, _, ok := runtime.Caller(0)
	if !ok {
		t.Fatal("resolve runtime caller for managed relay test")
	}
	pulseRoot := filepath.Clean(filepath.Join(filepath.Dir(currentFile), "..", ".."))
	pulseProRoot := strings.TrimSpace(os.Getenv("PULSE_REPO_ROOT_PULSE_PRO"))
	hasExplicitPulseProRoot := pulseProRoot != ""
	if !hasExplicitPulseProRoot {
		pulseProRoot = filepath.Join(filepath.Dir(pulseRoot), "pulse-pro")
	}
	pulseProRelayDir := filepath.Join(pulseProRoot, "relay-server")
	if _, err := os.Stat(filepath.Join(pulseProRelayDir, "main.go")); err != nil {
		if hasExplicitPulseProRoot {
			t.Fatalf("managed relay runtime proof was explicitly configured with PULSE_REPO_ROOT_PULSE_PRO=%s, but relay-server is unavailable: %v", pulseProRoot, err)
		}
		t.Skipf("managed relay runtime proof requires sibling pulse-pro relay-server; skipping single-repository checkout (set PULSE_REPO_ROOT_PULSE_PRO to enforce): %v", err)
	}
	return pulseRoot, pulseProRelayDir
}

func buildManagedRelayBinary(t *testing.T, relayServerDir string) string {
	t.Helper()
	binaryPath := filepath.Join(t.TempDir(), "pulse-relay-managed-runtime")
	cmd := exec.Command("go", "build", "-o", binaryPath, ".")
	cmd.Dir = relayServerDir
	output, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("build managed relay server binary: %v\n%s", err, string(output))
	}
	return binaryPath
}

func signManagedRelayJWT(t *testing.T, privateKey ed25519.PrivateKey, claims map[string]any) string {
	t.Helper()
	header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"EdDSA","typ":"JWT"}`))
	payloadBytes, err := json.Marshal(claims)
	if err != nil {
		t.Fatalf("marshal managed relay claims: %v", err)
	}
	payload := base64.RawURLEncoding.EncodeToString(payloadBytes)
	signedData := []byte(header + "." + payload)
	signature := ed25519.Sign(privateKey, signedData)
	return header + "." + payload + "." + base64.RawURLEncoding.EncodeToString(signature)
}

func allocateLoopbackAddress(t *testing.T) string {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("allocate loopback port: %v", err)
	}
	addr := listener.Addr().String()
	if err := listener.Close(); err != nil {
		t.Fatalf("close loopback allocator: %v", err)
	}
	return addr
}

func startManagedRelayServer(
	t *testing.T,
	binaryPath string,
	addr string,
	dataDir string,
	publicKeyB64 string,
) *managedRelayServer {
	t.Helper()
	logBuf := &safeBuffer{}
	cmd := exec.Command(binaryPath)
	cmd.Env = append(os.Environ(),
		"PULSE_RELAY_ADDR="+addr,
		"PULSE_RELAY_PUBLIC_KEY="+publicKeyB64,
		"PULSE_RELAY_DATA_DIR="+dataDir,
		"PULSE_RELAY_DRAIN_TIMEOUT=2s",
	)
	cmd.Stdout = logBuf
	cmd.Stderr = logBuf
	if err := cmd.Start(); err != nil {
		t.Fatalf("start managed relay server: %v", err)
	}

	server := &managedRelayServer{
		cmd:     cmd,
		logBuf:  logBuf,
		addr:    addr,
		dataDir: dataDir,
	}
	t.Cleanup(func() {
		server.stopKill()
	})
	waitForManagedRelayHealth(t, addr, logBuf, 10*time.Second)
	return server
}

func (s *managedRelayServer) stopKill() {
	if s == nil || s.cmd == nil || s.cmd.Process == nil {
		return
	}
	if s.cmd.ProcessState != nil && s.cmd.ProcessState.Exited() {
		return
	}
	_ = s.cmd.Process.Kill()
	_, _ = s.cmd.Process.Wait()
}

func (s *managedRelayServer) stopTerm() {
	if s == nil || s.cmd == nil || s.cmd.Process == nil {
		return
	}
	if s.cmd.ProcessState != nil && s.cmd.ProcessState.Exited() {
		return
	}
	_ = s.cmd.Process.Signal(os.Interrupt)
	done := make(chan struct{})
	go func() {
		_, _ = s.cmd.Process.Wait()
		close(done)
	}()
	select {
	case <-done:
	case <-time.After(5 * time.Second):
		_ = s.cmd.Process.Kill()
		<-done
	}
}

func waitForManagedRelayHealth(t *testing.T, addr string, logBuf *safeBuffer, timeout time.Duration) {
	t.Helper()
	deadline := time.Now().Add(timeout)
	url := "http://" + addr + "/healthz"
	for time.Now().Before(deadline) {
		resp, err := http.Get(url)
		if err == nil {
			body, _ := io.ReadAll(resp.Body)
			_ = resp.Body.Close()
			if resp.StatusCode == http.StatusOK && strings.Contains(string(body), `"status":"ok"`) {
				return
			}
		}
		time.Sleep(100 * time.Millisecond)
	}
	t.Fatalf("managed relay server at %s never became healthy\nserver logs:\n%s", addr, logBuf.String())
}

func waitForManagedRelayConnected(t *testing.T, client *Client, timeout time.Duration) string {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		status := client.Status()
		if status.Connected && status.InstanceID != "" {
			return status.InstanceID
		}
		time.Sleep(100 * time.Millisecond)
	}
	t.Fatalf("timed out waiting for managed relay client connection; final status=%+v", client.Status())
	return ""
}

func waitForManagedRelayIdle(t *testing.T, client *Client, timeout time.Duration) {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		status := client.Status()
		if status.Connected && status.ActiveChannels == 0 {
			return
		}
		time.Sleep(100 * time.Millisecond)
	}
	t.Fatalf("timed out waiting for managed relay client idle status; final status=%+v", client.Status())
}

func waitForManagedRelayLog(t *testing.T, logBuf *safeBuffer, needle string, timeout time.Duration) {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if strings.Contains(logBuf.String(), needle) {
			return
		}
		time.Sleep(100 * time.Millisecond)
	}
	t.Fatalf("timed out waiting for managed relay log %q\nlogs:\n%s", needle, logBuf.String())
}

func openManagedRelayAppConnection(t *testing.T, relayAddr string, instanceID string, authToken string) (*websocket.Conn, uint32) {
	t.Helper()
	conn, _, err := websocket.DefaultDialer.Dial("ws://"+relayAddr+"/ws/app", nil)
	if err != nil {
		t.Fatalf("dial managed relay app websocket: %v", err)
	}

	connectBytes, err := MarshalControlPayload(ConnectPayload{
		InstanceID: instanceID,
		AuthToken:  authToken,
	})
	if err != nil {
		t.Fatalf("marshal managed relay connect payload: %v", err)
	}
	frame := NewFrame(FrameConnect, 0, connectBytes)
	encoded, err := EncodeFrame(frame)
	if err != nil {
		t.Fatalf("encode managed relay connect frame: %v", err)
	}
	if err := conn.WriteMessage(websocket.BinaryMessage, encoded); err != nil {
		t.Fatalf("write managed relay connect frame: %v", err)
	}

	if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
		t.Fatalf("set managed relay app read deadline: %v", err)
	}
	_, msg, err := conn.ReadMessage()
	if err != nil {
		t.Fatalf("read managed relay connect ack: %v", err)
	}
	_ = conn.SetReadDeadline(time.Time{})
	ackFrame, err := DecodeFrame(msg)
	if err != nil {
		t.Fatalf("decode managed relay connect ack: %v", err)
	}
	if ackFrame.Type != FrameConnectAck {
		t.Fatalf("expected CONNECT_ACK from managed relay app websocket, got %s", FrameTypeName(ackFrame.Type))
	}
	var ack ConnectAckPayload
	if err := UnmarshalControlPayload(ackFrame.Payload, &ack); err != nil {
		t.Fatalf("unmarshal managed relay connect ack: %v", err)
	}
	return conn, ack.ChannelID
}

func sendManagedRelayProxyRequest(t *testing.T, conn *websocket.Conn, channelID uint32, request ProxyRequest) {
	t.Helper()
	payload, err := json.Marshal(request)
	if err != nil {
		t.Fatalf("marshal managed relay proxy request: %v", err)
	}
	frame := NewFrame(FrameData, channelID, payload)
	encoded, err := EncodeFrame(frame)
	if err != nil {
		t.Fatalf("encode managed relay proxy request: %v", err)
	}
	if err := conn.WriteMessage(websocket.BinaryMessage, encoded); err != nil {
		t.Fatalf("write managed relay proxy request: %v", err)
	}
}

func verifyManagedRelayProxyRoundTrip(t *testing.T, relayAddr string, instanceID string, path string) {
	t.Helper()
	conn, channelID := openManagedRelayAppConnection(t, relayAddr, instanceID, "managed-runtime-app-token")
	defer conn.Close()

	sendManagedRelayProxyRequest(t, conn, channelID, ProxyRequest{
		ID:     "req_roundtrip_" + strings.Trim(strings.ReplaceAll(path, "/", "_"), "_"),
		Method: "GET",
		Path:   path,
	})

	if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
		t.Fatalf("set managed relay roundtrip read deadline: %v", err)
	}
	_, msg, err := conn.ReadMessage()
	if err != nil {
		t.Fatalf("read managed relay roundtrip response: %v", err)
	}
	_ = conn.SetReadDeadline(time.Time{})
	frame, err := DecodeFrame(msg)
	if err != nil {
		t.Fatalf("decode managed relay roundtrip response: %v", err)
	}
	if frame.Type != FrameData {
		t.Fatalf("expected DATA from managed relay roundtrip, got %s", FrameTypeName(frame.Type))
	}
	var response ProxyResponse
	if err := json.Unmarshal(frame.Payload, &response); err != nil {
		t.Fatalf("unmarshal managed relay proxy response: %v", err)
	}
	if response.Status != http.StatusOK {
		t.Fatalf("managed relay proxy response status=%d body=%q", response.Status, response.Body)
	}
	bodyBytes, err := base64.StdEncoding.DecodeString(response.Body)
	if err != nil {
		t.Fatalf("decode managed relay proxy response body: %v", err)
	}
	if !strings.Contains(string(bodyBytes), `"endpoint":"status"`) {
		t.Fatalf("managed relay proxy response body=%s, want endpoint=status", string(bodyBytes))
	}
}