Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

11.7 KB
Raw Blame
//go:build unix

package api

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/json"
	"encoding/pem"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)

// This is an offline, synthetic process journey, not an installed-service or
// reporter recovery test. It joins the actual host issuer, installer token
// writer, agent CLI/transport and authenticated server routes. The terminal
// prompt and the rest of the privileged installer are outside this seam.
func TestHostAgentCredentialFileHandoff(t *testing.T) {
	root, err := filepath.Abs(filepath.Join("..", ".."))
	if err != nil {
		t.Fatal(err)
	}
	binary := filepath.Join(t.TempDir(), "pulse-agent")
	buildCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
	defer cancel()
	build := exec.CommandContext(buildCtx, "go", "build", "-o", binary, "./cmd/pulse-agent")
	build.Dir = root
	if output, err := build.CombinedOutput(); err != nil {
		t.Fatalf("build agent from this source: %v\n%s", err, output)
	}

	for _, tc := range []struct {
		name        string
		environment bool
		format      string
		wantStatus  int
		startupErr  bool
	}{
		{name: "file only", wantStatus: http.StatusOK},
		{name: "file over environment", environment: true, wantStatus: http.StatusOK},
		{name: "outer whitespace", format: "whitespace", wantStatus: http.StatusOK},
		{name: "quoted credential", format: "quoted", wantStatus: http.StatusUnauthorized},
		{name: "assignment credential", format: "assignment", wantStatus: http.StatusUnauthorized},
		{name: "different bare credential", format: "different", wantStatus: http.StatusUnauthorized},
		{name: "empty file refuses fallback", environment: true, format: "empty", startupErr: true},
		{name: "missing file refuses fallback", environment: true, format: "missing", startupErr: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			adminRaw := "synthetic-handoff-admin.12345678"
			envRaw := "synthetic-handoff-environment.12345678"
			admin := newTokenRecord(t, adminRaw, []string{config.ScopeWildcard}, nil)
			envRecord := newTokenRecord(t, envRaw, []string{config.ScopeAgentReport, config.ScopeAgentConfigRead}, nil)
			runtime := newHostRemovalLifecycleHTTPRuntime(t, t.TempDir(), []config.APITokenRecord{admin, envRecord})
			t.Cleanup(runtime.stop)

			issued := serveHostRemovalLifecycleRequest(t, runtime, http.MethodPost, "/api/agent-install-command", adminRaw,
				[]byte(`{"type":"host","enableCommands":false,"name":"credential-handoff"}`))
			install := decodeHostAgentInstallResponse(t, issued)
			if install.Token == "" || install.Record == nil {
				t.Fatal("host issuer did not return a scoped credential")
			}
			assertHandoffCredentialUse(t, runtime.config, install.Record.ID, false)
			config.Mu.RLock()
			for _, record := range runtime.config.APITokens {
				if record.ID == install.Record.ID && (record.HasScope(config.ScopeAgentExec) || !record.HasScope(config.ScopeAgentReport) || !record.HasScope(config.ScopeAgentConfigRead)) {
					t.Error("host issuer changed monitoring-only scopes")
				}
			}
			config.Mu.RUnlock()

			events := make(chan handoffHTTPEvent, 64)
			var requests atomic.Int64
			server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				requests.Add(1)
				event := handoffHTTPEvent{path: r.URL.Path}
				if strings.HasSuffix(r.URL.Path, "/report") {
					body, readErr := io.ReadAll(io.LimitReader(r.Body, 4<<20))
					if readErr != nil {
						t.Error("could not observe synthetic report")
					}
					r.Body = io.NopCloser(bytes.NewReader(body))
					// Observe the real wire encoding without changing the body
					// delivered to the canonical router and decompressor.
					var reportBody io.Reader = bytes.NewReader(body)
					if r.Header.Get("Content-Encoding") == "gzip" {
						compressed, err := gzip.NewReader(reportBody)
						if err != nil {
							t.Error("could not observe compressed synthetic report")
							w.WriteHeader(http.StatusBadRequest)
							return
						}
						defer compressed.Close()
						reportBody = compressed
					}
					if err := json.NewDecoder(io.LimitReader(reportBody, 4<<20)).Decode(&event.report); err != nil {
						t.Error("could not decode synthetic report")
					}
				}
				rec := httptest.NewRecorder()
				runtime.router.Handler().ServeHTTP(rec, r)
				for key, values := range rec.Header() {
					w.Header()[key] = values
				}
				w.WriteHeader(rec.Code)
				_, _ = w.Write(rec.Body.Bytes())
				event.status = rec.Code
				if strings.HasSuffix(event.path, "/lookup") || strings.HasSuffix(event.path, "/report") {
					select {
					case events <- event:
					default:
						t.Error("synthetic HTTP event buffer exceeded")
					}
				}
			}))
			t.Cleanup(server.Close)

			state := t.TempDir()
			tokenFile := persistHandoffToken(t, root, state, install.Token)
			if tc.format != "" {
				content := install.Token
				switch tc.format {
				case "whitespace":
					content = " \t" + content + "\r\n"
				case "quoted":
					content = `"` + content + `"`
				case "assignment":
					content = "PULSE_TOKEN=" + content
				case "different":
					content = "different-synthetic-token"
				case "empty", "missing":
					content = ""
				}
				// Adverse file representations are deliberate fixture edits, not
				// claims that the generator or installer writes these formats.
				if err := os.WriteFile(tokenFile, []byte(content), 0600); err != nil {
					t.Fatal(err)
				}
				if tc.format == "missing" {
					if err := os.Remove(tokenFile); err != nil {
						t.Fatal(err)
					}
				}
			}
			caFile := filepath.Join(t.TempDir(), "synthetic-server-ca.pem")
			if err := os.WriteFile(caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0600); err != nil {
				t.Fatal(err)
			}
			environment := ""
			if tc.environment {
				environment = envRaw
			}
			runs := 1
			if tc.wantStatus == http.StatusOK {
				runs = 2 // Same state, file and identity after an ordinary restart.
			}
			for run := 0; run < runs; run++ {
				stop, exit, logs := startHandoffAgent(t, binary, server.URL, caFile, tokenFile, state, environment)
				if tc.startupErr {
					select {
					case err := <-exit:
						if err == nil || !strings.Contains(logs.String(), "--token-file") {
							t.Fatal("invalid selected file did not produce a startup error")
						}
					case <-time.After(15 * time.Second):
						t.Fatal("invalid selected file did not stop startup")
					}
					if requests.Load() != 0 {
						t.Fatal("invalid selected file sent an HTTP request using fallback authority")
					}
				} else {
					awaitHandoffHTTP(t, events, tc.wantStatus, run > 0)
				}
				stop()
				if !tc.startupErr {
					if err := <-exit; err != nil {
						t.Fatal("synthetic agent did not stop cleanly")
					}
				}
				if strings.Contains(logs.String(), install.Token) || strings.Contains(logs.String(), envRaw) {
					t.Fatal("agent logs disclosed a credential")
				}
			}
			assertHandoffCredentialUse(t, runtime.config, install.Record.ID, tc.wantStatus == http.StatusOK)
			assertHandoffCredentialUse(t, runtime.config, envRecord.ID, false)
			if tc.wantStatus == http.StatusOK && len(runtime.monitor.GetState().Hosts) != 1 {
				t.Fatal("process restart duplicated the host identity")
			}
			t.Logf("runs=%d report_status=%d startup_refused=%v selected_record_used=%v environment_record_used=false", runs, tc.wantStatus, tc.startupErr, tc.wantStatus == http.StatusOK)
		})
	}
}

type handoffHTTPEvent struct {
	path   string
	status int
	report agentshost.Report
}

func awaitHandoffHTTP(t *testing.T, events <-chan handoffHTTPEvent, wantStatus int, restarted bool) {
	t.Helper()
	lookup, report := false, false
	deadline := time.NewTimer(30 * time.Second)
	defer deadline.Stop()
	for !lookup || !report {
		select {
		case event := <-events:
			if strings.HasSuffix(event.path, "/lookup") {
				lookup = true
				// A new valid identity is not registered until its first report.
				if event.status != wantStatus && !(wantStatus == http.StatusOK && !restarted && event.status == http.StatusNotFound) {
					t.Fatalf("lookup status = %d, want authenticated lookup or new-identity absence", event.status)
				}
			} else {
				report = true
				if event.status != wantStatus {
					t.Fatalf("report status = %d, want %d", event.status, wantStatus)
				}
				if event.report.Agent.ID != "credential-handoff-agent" || event.report.Host.Hostname != "credential-handoff.local" || event.report.Agent.CommandsEnabled {
					t.Fatal("report changed selected identity or command authority")
				}
			}
		case <-deadline.C:
			t.Fatal("agent did not complete both lookup and report")
		}
	}
}

func assertHandoffCredentialUse(t *testing.T, cfg *config.Config, id string, used bool) {
	t.Helper()
	config.Mu.RLock()
	defer config.Mu.RUnlock()
	for _, record := range cfg.APITokens {
		if record.ID == id {
			if (record.LastUsedAt != nil) != used {
				t.Fatal("credential last-used evidence does not match authentication")
			}
			return
		}
	}
	t.Fatal("credential record missing")
}

func persistHandoffToken(t *testing.T, root, state, token string) string {
	t.Helper()
	script, err := os.ReadFile(filepath.Join(root, "scripts", "install.sh"))
	if err != nil {
		t.Fatal(err)
	}
	text := string(script)
	start := strings.Index(text, "ensure_runtime_token_file() {")
	end := strings.Index(text, "\nclear_proxmox_state_if_needed() {")
	if start < 0 || end <= start {
		t.Fatal("canonical installer token writer not found")
	}
	cmd := exec.Command("bash", "-c", `set -euo pipefail
TMP_FILES=(); ENROLL=false; EXIT_GENERAL=1
log_info() { :; }; fail() { exit 1; }
PULSE_TOKEN=$(cat)
`+text[start:end]+`ensure_runtime_token_file "$STATE_DIR"`)
	cmd.Env = append(os.Environ(), "STATE_DIR="+state)
	cmd.Stdin = strings.NewReader(token)
	if output, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("private installer token writer: %v (output bytes %d)", err, len(output))
	}
	tokenFile := filepath.Join(state, "token")
	for path, mode := range map[string]os.FileMode{state: 0700, tokenFile: 0600} {
		info, err := os.Stat(path)
		if err != nil || info.Mode().Perm() != mode {
			t.Fatal("installer token writer did not retain private file/parent modes")
		}
	}
	return tokenFile
}

func startHandoffAgent(t *testing.T, binary, url, caFile, tokenFile, state, envToken string) (func(), <-chan error, *bytes.Buffer) {
	t.Helper()
	cmd := exec.Command(binary, "--url", url, "--cacert", caFile, "--token-file", tokenFile,
		"--state-dir", state, "--agent-id", "credential-handoff-agent", "--hostname", "credential-handoff.local",
		"--enable-host", "--enable-docker=false", "--enable-kubernetes=false", "--enable-proxmox=false",
		"--enable-commands=false", "--command-authority", "monitoring-only", "--disable-auto-update", "--interval", "1s", "--health-addr", "127.0.0.1:0")
	for _, value := range os.Environ() {
		if !strings.HasPrefix(value, "PULSE_") {
			cmd.Env = append(cmd.Env, value)
		}
	}
	cmd.Env = append(cmd.Env, "PULSE_TOKEN="+envToken)
	var logs bytes.Buffer
	cmd.Stdout, cmd.Stderr = &logs, &logs
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	if err := cmd.Start(); err != nil {
		t.Fatal("start synthetic agent")
	}
	exit := make(chan error, 1)
	done := make(chan struct{})
	go func() { exit <- cmd.Wait(); close(done) }()
	var once sync.Once
	stop := func() {
		once.Do(func() {
			select {
			case <-done:
				return
			default:
			}
			_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM)
			select {
			case <-done:
			case <-time.After(10 * time.Second):
				_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
				<-done
				t.Error("synthetic agent required forced cleanup")
			}
		})
	}
	t.Cleanup(stop)
	return stop, exit, &logs
}