Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

16.9 KB
Raw Blame
package api

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth"
)

const agentEventLifecycleToken = "synthetic-event-lifecycle-token.12345678"
const agentEventControlToken = "synthetic-event-control-token.12345678"

type agentLifecycleStream struct {
	reader *bufio.Reader
}

func openAgentLifecycleStream(t *testing.T, server *httptest.Server, raw, transport string) *agentLifecycleStream {
	t.Helper()
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	t.Cleanup(cancel)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/api/agent/events", nil)
	if err != nil {
		t.Fatal(err)
	}
	if transport == "session" {
		req.AddCookie(&http.Cookie{Name: cookieNameSession, Value: raw})
	} else if transport == "bearer" {
		req.Header.Set("Authorization", "Bearer "+raw)
	} else {
		req.Header.Set("X-API-Token", raw)
	}
	resp, err := server.Client().Do(req)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { resp.Body.Close() })
	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		t.Fatalf("stream admission = %d: %s", resp.StatusCode, body)
	}
	stream := &agentLifecycleStream{reader: bufio.NewReader(resp.Body)}
	if first := stream.next(t); first.Kind != AgentEventStreamConnected {
		t.Fatalf("first frame = %q, want connected", first.Kind)
	}
	return stream
}

func (s *agentLifecycleStream) read() (AgentEvent, error) {
	for {
		line, err := s.reader.ReadString('\n')
		if err != nil {
			return AgentEvent{}, err
		}
		if strings.HasPrefix(line, "data: ") {
			var event AgentEvent
			err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &event)
			return event, err
		}
	}
}

func (s *agentLifecycleStream) next(t *testing.T) AgentEvent {
	t.Helper()
	for {
		event, err := s.read()
		if err != nil {
			t.Fatalf("read stream frame: %v", err)
		}
		if event.Kind != AgentEventHeartbeat {
			return event
		}
	}
}

func (s *agentLifecycleStream) wantClosed(t *testing.T) {
	t.Helper()
	for {
		event, err := s.read()
		if errors.Is(err, io.EOF) {
			return
		}
		if err != nil {
			t.Fatalf("stream did not close cleanly after authority withdrawal: %v", err)
		}
		// A heartbeat admitted before the fixture mutation may already be in
		// flight. It must not hide a product frame or substitute for EOF.
		if event.Kind != AgentEventHeartbeat {
			t.Fatalf("withdrawn credential still received %q", event.Kind)
		}
	}
}

func newAgentLifecycleRouter(t *testing.T, heartbeat time.Duration) (*Router, *config.Config, *httptest.Server) {
	t.Helper()
	primary := newTokenRecord(t, agentEventLifecycleToken, []string{config.ScopeMonitoringRead, config.ScopeAIExecute}, nil)
	control := newTokenRecord(t, agentEventControlToken, []string{config.ScopeWildcard}, nil)
	cfg := newTestConfigWithTokens(t, primary, control)
	router := NewRouter(cfg, nil, nil, nil, nil, "test")
	cleanupTestRouter(t, router)
	router.agentEventBroadcaster.heartbeatInterval = heartbeat
	server := httptest.NewServer(router.Handler())
	t.Cleanup(server.Close)
	return router, cfg, server
}

type agentEventRefreshWriter struct {
	*httptest.ResponseRecorder
	afterFlush func()
}

func (w *agentEventRefreshWriter) Flush() {
	w.ResponseRecorder.Flush()
	w.afterFlush()
}

// Exercise the actual stream's refresh/redaction loop, without timing or heap
// thresholds. Each flush schedules exactly one next frame on the same goroutine;
// the last cancels the original context. Inspect only the exported parent field
// of WithValue's context to prove it cannot retain earlier refresh snapshots.
func TestAgentEventTokenRefreshKeepsBoundedContext(t *testing.T) {
	for _, transport := range []string{"header", "bearer"} {
		t.Run(transport, func(t *testing.T) {
			const orgID = "refresh-tenant"
			token := newTokenRecord(t, agentEventLifecycleToken, []string{config.ScopeMonitoringRead, config.ScopeAIExecute}, nil)
			token.OrgID = orgID
			cfg := newTestConfigWithTokens(t, token)
			// The deadline is a preserved request value, not a throughput budget.
			// Explicit cancellation after the fixed frame count owns termination;
			// the enclosing go test timeout still bounds a hung handler.
			ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
			defer cancel()
			type requestMarker struct{}
			ctx = context.WithValue(ctx, requestMarker{}, "admitted-value")
			ctx = context.WithValue(ctx, OrgIDContextKey, orgID)
			ctx = internalauth.WithUser(ctx, apiTokenAuthenticatedUser(&token))
			req := httptest.NewRequest(http.MethodGet, "/api/agent/events", nil).WithContext(ctx)
			if transport == "bearer" {
				req.Header.Set("Authorization", "Bearer "+agentEventLifecycleToken)
			} else {
				req.Header.Set("X-API-Token", agentEventLifecycleToken)
			}
			attachAPITokenRecord(req, &token)
			admittedContext := req.Context()
			admittedToken := internalauth.GetAPIToken(admittedContext)
			deadline, _ := admittedContext.Deadline()
			broadcaster := NewAgentEventBroadcaster()
			broadcaster.heartbeatInterval = time.Hour
			router := &Router{config: cfg, agentEventBroadcaster: broadcaster}
			record := agentDisclosureRecord(t, false)
			before, err := json.Marshal(record)
			if err != nil {
				t.Fatal(err)
			}
			const productFrames = 512
			flushed := 0
			restricted := false
			writer := &agentEventRefreshWriter{ResponseRecorder: httptest.NewRecorder()}
			writer.afterFlush = func() {
				// WithValue must be one fresh layer over the original context,
				// never over the layer created by the previous frame/heartbeat.
				parent := reflect.ValueOf(req.Context()).Elem().FieldByName("Context")
				if !parent.IsValid() || !parent.CanInterface() || parent.Interface() != admittedContext {
					t.Fatalf("refresh %d retained a previous token context instead of the admitted context", flushed+1)
				}
				if req.Context().Done() != admittedContext.Done() || req.Context().Value(requestMarker{}) != "admitted-value" ||
					GetOrgID(req.Context()) != orgID || internalauth.GetUser(req.Context()) != apiTokenAuthenticatedUser(&token) {
					t.Fatal("refresh lost admitted cancellation, request value, organization or principal")
				}
				if got, ok := req.Context().Deadline(); !ok || !got.Equal(deadline) {
					t.Fatal("refresh changed the admitted deadline")
				}
				snapshot := getAPITokenRecordFromRequest(req)
				if snapshot == nil || snapshot.ID != token.ID || snapshot.HasScope(config.ScopeAIExecute) == restricted {
					t.Fatal("refresh lost admitted identity or current disclosure scopes")
				}
				stream := &agentLifecycleStream{reader: bufio.NewReader(strings.NewReader(writer.Body.String()))}
				event, err := stream.read()
				if err != nil {
					t.Fatal(err)
				}
				writer.Body.Reset()
				if flushed == 0 {
					if event.Kind != AgentEventStreamConnected {
						t.Fatalf("first frame = %q, want connected", event.Kind)
					}
				} else if flushed%2 == 1 {
					if event.Kind != AgentEventApprovalPending {
						t.Fatalf("frame = %q, want approval", event.Kind)
					}
					var approval AgentEventApprovalPendingPayload
					body, _ := json.Marshal(event.Payload)
					if err := json.Unmarshal(body, &approval); err != nil {
						t.Fatal(err)
					}
					if approval.CommandRedacted != restricted || (approval.Command == "") != restricted {
						t.Fatal("approval did not use current disclosure scopes")
					}
				} else {
					if event.Kind != AgentEventActionCompleted {
						t.Fatalf("frame = %q, want action", event.Kind)
					}
					body, _ := json.Marshal(event)
					assertAgentDisclosureWire(t, body, restricted)
				}
				if flushed == productFrames {
					cancel()
					return
				}
				// Change scopes before the next pair of approval/action frames.
				restricted = (flushed/2)%2 == 1
				config.Mu.Lock()
				cfg.APITokens[0].Scopes = []string{config.ScopeMonitoringRead}
				if !restricted {
					cfg.APITokens[0].Scopes = append(cfg.APITokens[0].Scopes, config.ScopeAIExecute)
				}
				config.Mu.Unlock()
				if flushed%2 == 0 {
					broadcaster.PublishApprovalPending(AgentEventApprovalPendingPayload{ApprovalID: "refresh-approval", Command: agentPrivateDetail})
				} else {
					broadcaster.PublishActionCompletedRecord(record)
				}
				flushed++
			}
			router.handleAgentEvents(writer, req)
			if flushed != productFrames || broadcaster.SubscriberCount() != 0 || req.Context().Err() != context.Canceled {
				t.Fatalf("stream did not finish all frames and unsubscribe on original cancellation: frames=%d subscribers=%d err=%v", flushed, broadcaster.SubscriberCount(), req.Context().Err())
			}
			if !admittedToken.HasScope(config.ScopeAIExecute) {
				t.Fatal("live refresh mutated the admitted token snapshot")
			}
			after, _ := json.Marshal(record)
			if !bytes.Equal(before, after) {
				t.Fatal("live redaction changed canonical evidence")
			}
			t.Logf("%d product frames: bounded context parent, live approval/action redaction and original cancellation verified", flushed)
		})
	}
}

func TestAgentEventTokenLifecycleRejectsWithdrawnAuthority(t *testing.T) {
	for _, transport := range []string{"header", "bearer"} {
		for _, change := range []string{"revoked", "expired", "read_scope_removed", "org_rebound", "identity_replaced", "principal_replaced", "invalid_role"} {
			t.Run(transport+"/"+change, func(t *testing.T) {
				router, cfg, server := newAgentLifecycleRouter(t, time.Hour)
				primary := openAgentLifecycleStream(t, server, agentEventLifecycleToken, transport)
				control := openAgentLifecycleStream(t, server, agentEventControlToken, transport)
				config.Mu.Lock()
				switch change {
				case "revoked":
					cfg.RemoveAPIToken(cfg.APITokens[0].ID)
				case "expired":
					past := time.Now().UTC().Add(-time.Second)
					cfg.APITokens[0].ExpiresAt = &past
				case "read_scope_removed":
					cfg.APITokens[0].Scopes = []string{config.ScopeSettingsRead}
				case "org_rebound":
					cfg.APITokens[0].OrgID = "another-org"
				case "identity_replaced":
					cfg.APITokens[0].ID = "replacement-identity"
				case "principal_replaced":
					cfg.APITokens[0].Metadata = map[string]string{apiTokenMetadataOwnerUserID: "replacement-owner"}
				case "invalid_role":
					cfg.APITokens[0].Metadata = map[string]string{internalauth.RuntimeRoleMetadataKey: internalauth.RuntimeRoleActionRunner}
				}
				config.Mu.Unlock()

				router.agentEventBroadcaster.PublishActionCompletedRecord(agentDisclosureRecord(t, false))
				primary.wantClosed(t)
				event := control.next(t)
				if event.Kind != AgentEventActionCompleted {
					t.Fatalf("independent subscriber lost action: %q", event.Kind)
				}
				body, _ := json.Marshal(event)
				assertAgentDisclosureWire(t, body, false)
				if router.agentEventBroadcaster.SubscriberCount() != 1 {
					t.Fatal("withdrawn stream did not release its subscription")
				}
			})
		}
	}
}

func TestAgentEventTokenLifecycleClosesIdleStream(t *testing.T) {
	for _, change := range []string{"revoke", "natural_expiry"} {
		t.Run(change, func(t *testing.T) {
			_, cfg, server := newAgentLifecycleRouter(t, 10*time.Millisecond)
			primary := openAgentLifecycleStream(t, server, agentEventLifecycleToken, "header")
			config.Mu.Lock()
			if change == "revoke" {
				cfg.RemoveAPIToken(cfg.APITokens[0].ID)
			} else {
				expires := time.Now().UTC().Add(30 * time.Millisecond)
				cfg.APITokens[0].ExpiresAt = &expires
			}
			config.Mu.Unlock()
			primary.wantClosed(t)
		})
	}
}

func TestAgentEventTokenLifecycleResolvesLiveTenantInventory(t *testing.T) {
	const orgID = "tenant-a"
	for _, change := range []string{"replaced", "removed", "same_credential_in_new_owner", "changes_during_validation", "no_global_fallback"} {
		t.Run(change, func(t *testing.T) {
			record := newTokenRecord(t, agentEventLifecycleToken, []string{config.ScopeMonitoringRead, config.ScopeAIExecute}, nil)
			record.OrgID = orgID
			base := &config.Config{}
			mtp := config.NewMultiTenantPersistence(t.TempDir())
			if _, err := mtp.GetPersistence(orgID); err != nil {
				t.Fatal(err)
			}
			makeManager := func(cfg *config.Config) *monitoring.MultiTenantMonitor {
				manager := monitoring.NewMultiTenantMonitor(base, mtp, nil)
				monitor := &monitoring.Monitor{}
				setUnexportedField(t, monitor, "config", cfg)
				setUnexportedField(t, manager, "monitors", map[string]*monitoring.Monitor{orgID: monitor})
				t.Cleanup(manager.Stop)
				return manager
			}
			old := makeManager(&config.Config{APITokens: []config.APITokenRecord{record}})
			nextCfg := &config.Config{}
			if change == "same_credential_in_new_owner" {
				nextCfg.APITokens = []config.APITokenRecord{record.Clone()}
				nextCfg.APITokens[0].Scopes = []string{config.ScopeMonitoringRead}
			}
			next := makeManager(nextCfg)
			req := httptest.NewRequest(http.MethodGet, "/api/agent/events", nil)
			req.Header.Set("X-Pulse-Org-ID", orgID)
			req.Header.Set("X-API-Token", agentEventLifecycleToken)
			req = extractAndStoreAuthContext(base, old, req)
			req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, orgID))
			source := req.Context().Value(tenantTokenSourceContextKey).(tenantTokenSource)
			currentManager := old
			resolutions := 0
			source.resolveManager = func() *monitoring.MultiTenantMonitor {
				resolutions++
				if change == "changes_during_validation" && resolutions%2 == 0 {
					return next
				}
				return currentManager
			}
			req = req.WithContext(context.WithValue(req.Context(), tenantTokenSourceContextKey, source))
			validate := agentEventTokenRevalidator(base, req)
			if change != "changes_during_validation" && !validate() {
				t.Fatal("initial tenant credential rejected")
			}
			currentManager = next
			if change == "removed" {
				currentManager = nil
			}
			if change == "changes_during_validation" {
				currentManager = old
			}
			if change == "no_global_fallback" {
				base.APITokens = []config.APITokenRecord{record.Clone()}
			}
			want := change == "same_credential_in_new_owner"
			if got := validate(); got != want {
				t.Fatalf("live tenant validation = %v, want %v", got, want)
			}
			if want && requestCanReadAgentCommandPayloads(req) {
				t.Fatal("live owner scope removal left stale command disclosure")
			}
		})
	}
}

func TestAgentEventTokenLifecyclePreservesSessionDisclosure(t *testing.T) {
	router, _, server := newAgentLifecycleRouter(t, time.Hour)
	const session = "synthetic-event-browser-session"
	GetSessionStore().CreateSession(session, time.Hour, "fixture", "127.0.0.1", "operator")
	t.Cleanup(func() { GetSessionStore().DeleteSession(session) })
	stream := openAgentLifecycleStream(t, server, session, "session")
	router.agentEventBroadcaster.PublishActionCompletedRecord(agentDisclosureRecord(t, false))
	body, _ := json.Marshal(stream.next(t))
	assertAgentDisclosureWire(t, body, false)
}

type agentLifecycleMCPClient struct {
	client *http.Client
	ready  chan struct{}
}

func (c agentLifecycleMCPClient) Do(req *http.Request) (*http.Response, error) {
	resp, err := c.client.Do(req)
	close(c.ready)
	return resp, err
}

// Use the same SSE-to-MCP consumer the shipped pulse-mcp adapter calls, but
// against the real registered API route, not a fixture-only SSE producer.
func TestAgentEventTokenLifecycleMCPConsumer(t *testing.T) {
	router, cfg, server := newAgentLifecycleRouter(t, time.Hour)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	ready := make(chan struct{})
	notifications := make(chan agentcapabilities.JSONRPCRequest, 4)
	done := make(chan error, 1)
	go func() {
		done <- agentcapabilities.StreamMCPEventNotifications(ctx, agentLifecycleMCPClient{server.Client(), ready}, server.URL, agentcapabilities.AgentEventsPath, agentEventLifecycleToken, func(n agentcapabilities.JSONRPCRequest) error {
			notifications <- n
			return nil
		})
	}()
	select {
	case <-ready:
	case <-ctx.Done():
		t.Fatal("MCP stream never connected")
	}
	router.agentEventBroadcaster.PublishFindingCreated("", AgentEventFindingCreatedPayload{FindingID: "before-revocation"})
	select {
	case n := <-notifications:
		if n.Method != "notifications/finding.created" || !strings.Contains(string(n.Params), "before-revocation") {
			t.Fatalf("valid MCP notification changed: %+v", n)
		}
	case <-ctx.Done():
		t.Fatal("valid MCP notification never arrived")
	}
	config.Mu.Lock()
	cfg.RemoveAPIToken(cfg.APITokens[0].ID)
	config.Mu.Unlock()
	router.agentEventBroadcaster.PublishActionCompletedRecord(agentDisclosureRecord(t, false))
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("withdrawn MCP stream did not reach clean EOF: %v", err)
		}
	case <-ctx.Done():
		t.Fatal("withdrawn MCP stream stayed open")
	}
	if len(notifications) != 0 {
		t.Fatal("MCP forwarded an event after revocation")
	}
	err := agentcapabilities.StreamMCPEventNotifications(ctx, server.Client(), server.URL, agentcapabilities.AgentEventsPath, agentEventLifecycleToken, nil)
	if err == nil || !strings.Contains(err.Error(), "status 401") {
		t.Fatalf("MCP reconnect did not retain revocation: %v", err)
	}
}