Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

14.6 KB
Raw Blame
package api

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math/rand/v2"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
	"github.com/rs/zerolog/log"
)

const (
	hostedEntitlementRefreshDefaultInterval = 2 * time.Hour
	hostedEntitlementRefreshMinInterval     = 15 * time.Minute
	hostedEntitlementRefreshMaxInterval     = 6 * time.Hour
	hostedEntitlementRefreshImmediateWindow = 30 * time.Minute
	hostedEntitlementRefreshBackoffMin      = 30 * time.Second
	hostedEntitlementRefreshBackoffMax      = 30 * time.Minute
	hostedEntitlementRefreshJitter          = 0.2
)

type hostedEntitlementRefreshLoop struct {
	mu      sync.Mutex
	cancel  context.CancelFunc
	wg      sync.WaitGroup
	running bool
}

type hostedEntitlementRefreshError struct {
	statusCode int
	message    string
	permanent  bool
}

func (e *hostedEntitlementRefreshError) Error() string {
	if e == nil {
		return ""
	}
	if strings.TrimSpace(e.message) != "" {
		return e.message
	}
	return fmt.Sprintf("hosted entitlement refresh failed with status %d", e.statusCode)
}

func (h *LicenseHandlers) ensureHostedEntitlementRefreshForOrg(orgID string, service *licenseService) {
	if h == nil || h.mtPersistence == nil {
		return
	}
	requestedOrgID := normalizeHostedEntitlementOrgID(orgID)
	effectiveOrgID, state, err := h.loadHostedEntitlementRefreshTarget(requestedOrgID)
	if err != nil {
		log.Warn().Err(err).Str("org_id", requestedOrgID).Msg("Failed to load billing state for hosted entitlement refresh")
		return
	}
	if service != nil && service.Current() != nil {
		h.stopHostedEntitlementRefreshLoop(effectiveOrgID)
		if effectiveOrgID != requestedOrgID {
			h.stopHostedEntitlementRefreshLoop(requestedOrgID)
		}
		return
	}
	if !hasHostedEntitlementRefreshToken(state) {
		h.stopHostedEntitlementRefreshLoop(effectiveOrgID)
		if effectiveOrgID != requestedOrgID {
			h.stopHostedEntitlementRefreshLoop(requestedOrgID)
		}
		return
	}

	loop := h.hostedEntitlementRefreshLoop(effectiveOrgID)
	if !loop.isRunning() && hostedEntitlementNeedsImmediateRefresh(state) {
		if _, permanent, err := h.refreshHostedEntitlementLeaseOnce(effectiveOrgID, service); err != nil {
			if permanent {
				log.Warn().Err(err).Str("org_id", effectiveOrgID).Msg("Permanent hosted entitlement refresh failure during initialization")
				h.stopHostedEntitlementRefreshLoop(effectiveOrgID)
				return
			}
			log.Warn().Err(err).Str("org_id", effectiveOrgID).Msg("Hosted entitlement refresh initialization failed")
		}
	}

	h.startHostedEntitlementRefreshLoop(effectiveOrgID)
	if effectiveOrgID != requestedOrgID {
		h.stopHostedEntitlementRefreshLoop(requestedOrgID)
	}
}

func hasHostedEntitlementRefreshToken(state *billingState) bool {
	return state != nil && strings.TrimSpace(state.EntitlementRefreshToken) != ""
}

func (h *LicenseHandlers) loadHostedEntitlementRefreshTarget(orgID string) (string, *billingState, error) {
	requestedOrgID := normalizeHostedEntitlementOrgID(orgID)
	if h == nil || h.mtPersistence == nil {
		return requestedOrgID, nil, nil
	}

	state, effectiveOrgID, err := config.LoadEffectiveEntitlementBillingState(h.mtPersistence.BaseDataDir(), requestedOrgID)
	if err != nil {
		return "", nil, err
	}

	var normalized *billingState
	if state != nil {
		normalized = normalizeBillingStateFromLicensing(state)
	}
	return normalizeHostedEntitlementOrgID(effectiveOrgID), normalized, nil
}

func hostedEntitlementNeedsImmediateRefresh(state *billingState) bool {
	if state == nil || strings.TrimSpace(state.EntitlementRefreshToken) == "" {
		return false
	}
	leaseClaims, err := hostedEntitlementLeaseClaimsFromState(state)
	if err != nil || leaseClaims == nil || leaseClaims.ExpiresAt == nil {
		return true
	}
	return time.Until(leaseClaims.ExpiresAt.Time) <= hostedEntitlementRefreshImmediateWindow
}

func hostedEntitlementLeaseClaimsFromState(state *billingState) (*entitlementLeaseClaimsModel, error) {
	if state == nil || strings.TrimSpace(state.EntitlementJWT) == "" {
		return nil, fmt.Errorf("entitlement lease token is required")
	}
	publicKey, err := trialActivationPublicKeyFromLicensing()
	if err != nil {
		return nil, err
	}
	return parseEntitlementLeaseTokenFromLicensing(state.EntitlementJWT, publicKey, "")
}

func (h *LicenseHandlers) hostedEntitlementInstanceHost(state *billingState) string {
	if host := entitlementExpectedInstanceHost(h.cfg); host != "" {
		return host
	}
	claims, err := hostedEntitlementLeaseClaimsFromState(state)
	if err != nil || claims == nil {
		return ""
	}
	return normalizeHostForTrial(claims.InstanceHost)
}

func (h *LicenseHandlers) hostedEntitlementRefreshLoop(orgID string) *hostedEntitlementRefreshLoop {
	if h == nil {
		return nil
	}
	if loop, ok := h.hostedLeaseRefresh.Load(orgID); ok {
		if typed, ok := loop.(*hostedEntitlementRefreshLoop); ok {
			return typed
		}
	}
	loop := &hostedEntitlementRefreshLoop{}
	actual, _ := h.hostedLeaseRefresh.LoadOrStore(orgID, loop)
	if typed, ok := actual.(*hostedEntitlementRefreshLoop); ok {
		return typed
	}
	return loop
}

func (l *hostedEntitlementRefreshLoop) isRunning() bool {
	if l == nil {
		return false
	}
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.running
}

func (h *LicenseHandlers) startHostedEntitlementRefreshLoop(orgID string) {
	loop := h.hostedEntitlementRefreshLoop(orgID)
	if loop == nil {
		return
	}

	loop.mu.Lock()
	defer loop.mu.Unlock()
	if loop.running {
		return
	}

	ctx, cancel := context.WithCancel(context.Background())
	loop.cancel = cancel
	loop.running = true
	loop.wg.Add(1)
	go func() {
		defer func() {
			loop.mu.Lock()
			loop.running = false
			loop.mu.Unlock()
			loop.wg.Done()
		}()
		h.runHostedEntitlementRefreshLoop(ctx, orgID)
	}()
}

func (h *LicenseHandlers) stopHostedEntitlementRefreshLoop(orgID string) {
	if h == nil {
		return
	}
	value, ok := h.hostedLeaseRefresh.Load(orgID)
	if !ok {
		return
	}
	loop, ok := value.(*hostedEntitlementRefreshLoop)
	if !ok || loop == nil {
		h.hostedLeaseRefresh.Delete(orgID)
		return
	}

	loop.mu.Lock()
	if !loop.running {
		loop.mu.Unlock()
		h.hostedLeaseRefresh.Delete(orgID)
		return
	}
	loop.cancel()
	loop.running = false
	loop.mu.Unlock()
	loop.wg.Wait()
	h.hostedLeaseRefresh.Delete(orgID)
}

func (h *LicenseHandlers) runHostedEntitlementRefreshLoop(ctx context.Context, orgID string) {
	consecutiveFailures := 0
	for {
		interval, ok := h.nextHostedEntitlementRefreshInterval(orgID, consecutiveFailures)
		if !ok {
			return
		}

		select {
		case <-ctx.Done():
			return
		case <-time.After(interval):
		}

		refreshed, permanent, err := h.refreshHostedEntitlementLeaseOnce(orgID, nil)
		if err != nil {
			if permanent {
				log.Warn().Err(err).Str("org_id", orgID).Msg("Stopping hosted entitlement refresh loop after permanent failure")
				return
			}
			consecutiveFailures++
			log.Warn().
				Err(err).
				Str("org_id", orgID).
				Int("consecutive_failures", consecutiveFailures).
				Dur("next_retry", hostedEntitlementRefreshBackoff(consecutiveFailures)).
				Msg("Hosted entitlement refresh failed")
			continue
		}
		if !refreshed {
			return
		}
		consecutiveFailures = 0
	}
}

func (h *LicenseHandlers) nextHostedEntitlementRefreshInterval(orgID string, consecutiveFailures int) (time.Duration, bool) {
	if consecutiveFailures > 0 {
		return hostedEntitlementRefreshBackoff(consecutiveFailures), true
	}
	if h == nil || h.mtPersistence == nil {
		return 0, false
	}
	billingStore := config.NewFileBillingStore(h.mtPersistence.BaseDataDir())
	state, err := billingStore.GetBillingState(orgID)
	if err != nil || !hasHostedEntitlementRefreshToken(state) {
		return 0, false
	}

	claims, err := hostedEntitlementLeaseClaimsFromState(state)
	if err != nil || claims == nil || claims.ExpiresAt == nil {
		return time.Minute, true
	}

	remaining := time.Until(claims.ExpiresAt.Time)
	if remaining <= hostedEntitlementRefreshImmediateWindow {
		return time.Minute, true
	}
	interval := remaining / 2
	if interval < hostedEntitlementRefreshMinInterval {
		interval = hostedEntitlementRefreshMinInterval
	}
	if interval > hostedEntitlementRefreshMaxInterval {
		interval = hostedEntitlementRefreshMaxInterval
	}
	return withHostedEntitlementRefreshJitter(interval), true
}

func hostedEntitlementRefreshBackoff(consecutiveFailures int) time.Duration {
	if consecutiveFailures <= 0 {
		return hostedEntitlementRefreshDefaultInterval
	}
	backoff := hostedEntitlementRefreshBackoffMin * (1 << min(consecutiveFailures-1, 10))
	if backoff > hostedEntitlementRefreshBackoffMax {
		backoff = hostedEntitlementRefreshBackoffMax
	}
	return backoff
}

func withHostedEntitlementRefreshJitter(interval time.Duration) time.Duration {
	jitterRange := float64(interval) * hostedEntitlementRefreshJitter
	offset := (rand.Float64()*2 - 1) * jitterRange
	return interval + time.Duration(offset)
}

func (h *LicenseHandlers) refreshHostedEntitlementLeaseOnce(orgID string, service *licenseService) (bool, bool, error) {
	if h == nil || h.mtPersistence == nil {
		return false, true, nil
	}
	if service == nil {
		if value, ok := h.services.Load(orgID); ok {
			if typed, ok := value.(*licenseService); ok {
				service = typed
			}
		}
	}
	if service != nil && service.Current() != nil {
		return false, true, nil
	}

	billingStore := config.NewFileBillingStore(h.mtPersistence.BaseDataDir())
	effectiveOrgID, state, err := h.loadHostedEntitlementRefreshTarget(orgID)
	if err != nil {
		return false, false, fmt.Errorf("load hosted entitlement refresh target: %w", err)
	}
	if !hasHostedEntitlementRefreshToken(state) {
		return false, true, nil
	}

	instanceHost := h.hostedEntitlementInstanceHost(state)
	if instanceHost == "" {
		return false, true, fmt.Errorf("hosted entitlement instance host is unavailable")
	}

	response, err := h.requestHostedEntitlementLeaseRefresh(effectiveOrgID, instanceHost, state.EntitlementRefreshToken)
	if err != nil {
		var refreshErr *hostedEntitlementRefreshError
		if errors.As(err, &refreshErr) && refreshErr != nil && refreshErr.permanent {
			if clearErr := h.clearHostedEntitlementState(effectiveOrgID, billingStore); clearErr != nil {
				log.Warn().Err(clearErr).Str("org_id", effectiveOrgID).Msg("Failed to clear hosted entitlement state after permanent refresh failure")
			}
			if service != nil && service.Current() == nil {
				service.SetEvaluator(nil)
				_ = h.ensureEvaluatorForOrg(effectiveOrgID, service)
			}
			return false, true, err
		}
		return false, false, err
	}

	publicKey, err := trialActivationPublicKeyFromLicensing()
	if err != nil {
		return false, false, fmt.Errorf("load entitlement verification key: %w", err)
	}
	leaseClaims, err := verifyEntitlementLeaseTokenFromLicensing(response.EntitlementJWT, publicKey, instanceHost, time.Now().UTC())
	if err != nil {
		return false, false, fmt.Errorf("verify refreshed entitlement lease: %w", err)
	}
	if normalizeHostedEntitlementOrgID(leaseClaims.OrgID) != normalizeHostedEntitlementOrgID(effectiveOrgID) {
		return false, false, fmt.Errorf("refreshed entitlement lease org mismatch")
	}

	updated := normalizeBillingStateFromLicensing(state)
	updated.EntitlementJWT = strings.TrimSpace(response.EntitlementJWT)
	updated.Capabilities = []string{}
	updated.Limits = map[string]int64{}
	updated.MetersEnabled = []string{}
	updated.PlanVersion = ""
	updated.SubscriptionState = ""
	updated.TrialStartedAt = leaseClaims.TrialStartedAt
	updated.TrialEndsAt = nil
	updated.TrialExtendedAt = nil
	if err := billingStore.SaveBillingState(effectiveOrgID, updated); err != nil {
		return false, false, fmt.Errorf("save refreshed entitlement lease: %w", err)
	}
	if service != nil && service.Current() == nil {
		service.SetEvaluator(newLicenseEvaluatorForBillingStoreFromLicensing(billingStore, effectiveOrgID, 0, instanceHost))
	}
	return true, false, nil
}

func (h *LicenseHandlers) clearHostedEntitlementState(orgID string, billingStore *config.FileBillingStore) error {
	if billingStore == nil {
		return nil
	}
	existing, err := billingStore.GetBillingState(orgID)
	if err != nil {
		return err
	}
	if existing == nil {
		return nil
	}
	existing.Capabilities = []string{}
	existing.Limits = map[string]int64{}
	existing.MetersEnabled = []string{}
	existing.EntitlementJWT = ""
	existing.EntitlementRefreshToken = ""
	existing.PlanVersion = string(subscriptionStateExpiredValue)
	existing.SubscriptionState = subscriptionStateExpiredValue
	existing.TrialEndsAt = nil
	existing.TrialExtendedAt = nil
	return billingStore.SaveBillingState(orgID, existing)
}

func (h *LicenseHandlers) requestHostedEntitlementLeaseRefresh(orgID, instanceHost, refreshToken string) (*hostedTrialLeaseRefreshResponse, error) {
	if h == nil {
		return nil, &hostedEntitlementRefreshError{permanent: true, message: "license handlers unavailable"}
	}
	refreshURL := hostedEntitlementRefreshURLFromConfig(h.cfg)
	if refreshURL == "" {
		return nil, &hostedEntitlementRefreshError{permanent: true, message: "hosted entitlement refresh URL unavailable"}
	}

	payload, err := json.Marshal(hostedTrialLeaseRefreshRequest{
		OrgID:                   normalizeHostedEntitlementOrgID(orgID),
		InstanceHost:            strings.TrimSpace(instanceHost),
		EntitlementRefreshToken: strings.TrimSpace(refreshToken),
	})
	if err != nil {
		return nil, fmt.Errorf("marshal hosted entitlement refresh request: %w", err)
	}
	req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, refreshURL, bytes.NewReader(payload))
	if err != nil {
		return nil, fmt.Errorf("build hosted entitlement refresh request: %w", err)
	}
	req.Header.Set("Content-Type", "application/json")

	client := securityutil.NewRestrictedOutboundHTTPClient(5*time.Second, securityutil.RestrictedOutboundHTTPOptions{
		AllowedSchemes: []string{"http", "https"},
		AllowLoopback:  true,
	})
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("post hosted entitlement refresh request: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		permanent := resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusGone || resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusNotFound
		return nil, &hostedEntitlementRefreshError{
			statusCode: resp.StatusCode,
			message:    fmt.Sprintf("hosted entitlement refresh returned status %d", resp.StatusCode),
			permanent:  permanent,
		}
	}

	var response hostedTrialLeaseRefreshResponse
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return nil, fmt.Errorf("decode hosted entitlement refresh response: %w", err)
	}
	if strings.TrimSpace(response.EntitlementJWT) == "" {
		return nil, fmt.Errorf("hosted entitlement refresh response missing entitlement_jwt")
	}
	return &response, nil
}

func normalizeHostedEntitlementOrgID(raw string) string {
	orgID := strings.TrimSpace(raw)
	if orgID == "" {
		return "default"
	}
	return orgID
}