Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

63.2 KB
Raw Blame
package websocket

import (
	"compress/flate"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gorilla/websocket"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
	"github.com/rcourtman/pulse-go-rewrite/pkg/audit"
	"github.com/rcourtman/pulse-go-rewrite/pkg/auth"
	"github.com/rs/zerolog/log"
)

const (
	// maxWebSocketInboundMessageSize bounds both encoded and decoded inbound messages.
	maxWebSocketInboundMessageSize = 64 * 1024
	// maxWebSocketOrgIDLength keeps org IDs bounded to prevent oversized header/query abuse.
	maxWebSocketOrgIDLength = 64
	// inboundLimitQueryParam lets a browser declare the largest state frame it
	// can safely accept before the hub builds that connection's initial state.
	inboundLimitQueryParam = "max_message_bytes"
	// minAdvertisedInboundBytes keeps tiny or malicious values from turning
	// ordinary control messages into an unusable connection.
	minAdvertisedInboundBytes = 64 * 1024
	stateTooLargeMessageType  = "stateTooLarge"
	websocketHubComponent     = "websocket_hub"
	initialWelcomeDelay       = 500 * time.Millisecond
	initialStateMessageDelay  = 100 * time.Millisecond
	// clientReadDeadline is how long a client may stay silent before the server
	// drops it. Liveness is proven by ANY inbound frame (data, JSON heartbeat,
	// or protocol pong) — proxies and middleboxes (Cloudflare, AV products)
	// sometimes eat ping/pong control frames, so pongs must not be the only
	// accepted proof of life. Must be comfortably above clientPingPeriod and
	// the frontend's 30s JSON heartbeat, including a background tab throttled
	// to one heartbeat per minute.
	clientReadDeadline = 90 * time.Second
	// clientPingPeriod is how often writePump sends protocol pings. At 30s a
	// pong-only client survives two consecutive lost ping/pong round trips
	// within clientReadDeadline.
	clientPingPeriod = 30 * time.Second
)

type stateTooLargePayload struct {
	Supersedes    string `json:"supersedes"`
	Bytes         int    `json:"bytes,omitempty"`
	MaxBytes      int64  `json:"maxBytes"`
	ResourceCount int    `json:"resourceCount"`
	HydrateFrom   string `json:"hydrateFrom"`
}

// extractPeerIP extracts just the IP part from a RemoteAddr (host:port format)
func extractPeerIP(remoteAddr string) string {
	host, _, err := net.SplitHostPort(remoteAddr)
	if err != nil {
		// RemoteAddr might not have a port, try using it directly
		return remoteAddr
	}
	return host
}

// isValidPrivateOrigin checks if the origin is from a valid private network
func isValidPrivateOrigin(host string) bool {
	// Check localhost variations
	if host == "localhost" || host == "127.0.0.1" || host == "::1" {
		return true
	}

	// Check if it's a valid IP address
	ip := net.ParseIP(host)
	if ip != nil {
		// Check if it's a private IP
		return ip.IsLoopback() || ip.IsPrivate()
	}

	// Allow common local domain patterns but be more restrictive
	// Only allow if it's clearly a local domain
	if strings.HasSuffix(host, ".local") || strings.HasSuffix(host, ".lan") {
		// But not arbitrary subdomains that could be malicious
		parts := strings.Split(host, ".")
		if len(parts) <= 3 { // hostname.local or hostname.subdomain.local
			return true
		}
	}

	return false
}

// normalizeForwardedProto coerces forwarded proto values into the HTTP scheme space so that
// websocket upgrades coming through proxies that emit ws/wss continue to compare equal to the
// browser-sent Origin header (which is always http/https).
func normalizeForwardedProto(proto string, fallback string) string {
	if proto == "" {
		return fallback
	}

	// Some proxies send comma-separated proto chains; take the first hop.
	if comma := strings.IndexByte(proto, ','); comma != -1 {
		proto = proto[:comma]
	}

	cleaned := strings.TrimSpace(strings.ToLower(proto))
	switch cleaned {
	case "wss":
		return "https"
	case "ws":
		return "http"
	case "https", "http":
		return cleaned
	default:
		if cleaned != "" {
			return cleaned
		}
		return fallback
	}
}

func isValidWebSocketOrgID(orgID string) bool {
	if orgID == "" || orgID == "." || orgID == ".." {
		return false
	}
	if len(orgID) > maxWebSocketOrgIDLength {
		return false
	}
	if strings.TrimSpace(orgID) != orgID {
		return false
	}
	if strings.ContainsAny(orgID, `/\`) {
		return false
	}
	for _, r := range orgID {
		if r < 0x20 || r == 0x7f {
			return false
		}
	}
	return true
}

// SetAllowedOrigins sets the allowed origins for CORS
func (h *Hub) SetAllowedOrigins(origins []string) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.allowedOrigins = append([]string(nil), origins...)
}

// checkOrigin validates the origin against allowed origins
func (h *Hub) checkOrigin(r *http.Request) bool {
	origin := strings.TrimSpace(r.Header.Get("Origin"))
	if origin == "" {
		return false
	}

	h.mu.RLock()
	allowedOrigins := append([]string(nil), h.allowedOrigins...)
	h.mu.RUnlock()

	// Determine the actual origin (accounting for proxy headers).
	// Only trust X-Forwarded-* headers when the peer is a known trusted proxy,
	// consistent with how auth.go gates proxy header trust via isTrustedProxyIP.
	scheme := "http"
	if r.TLS != nil {
		scheme = "https"
	}

	h.mu.RLock()
	trustedProxyFn := h.isTrustedProxy
	h.mu.RUnlock()

	peerIP := extractPeerIP(r.RemoteAddr)
	peerIsTrusted := trustedProxyFn != nil && trustedProxyFn(peerIP)

	if peerIsTrusted {
		if forwardedProto := r.Header.Get("X-Forwarded-Proto"); forwardedProto != "" {
			scheme = normalizeForwardedProto(forwardedProto, scheme)
		} else if forwardedScheme := r.Header.Get("X-Forwarded-Scheme"); forwardedScheme != "" {
			scheme = normalizeForwardedProto(forwardedScheme, scheme)
		}
	}

	host := r.Host
	if peerIsTrusted {
		if forwardedHost := r.Header.Get("X-Forwarded-Host"); forwardedHost != "" {
			host = forwardedHost
		}
	}

	requestOrigin := scheme + "://" + host

	// Allow same-origin requests
	if origin == requestOrigin {
		return true
	}
	if securityutil.SameHostWebSocketOrigin(origin, host) {
		return true
	}

	// Check if wildcard is allowed
	for _, allowed := range allowedOrigins {
		if allowed == "*" {
			return true
		}
		if allowed == origin {
			return true
		}
	}

	// If no origins configured, only allow from truly private networks
	// SECURITY: This is a relaxed policy for homelab deployments. For production,
	// configure explicit allowed origins via WEBSOCKET_ALLOWED_ORIGINS env var.
	if len(allowedOrigins) == 0 {
		// Parse the origin URL to validate it properly
		originHost := origin
		if strings.HasPrefix(origin, "http://") {
			originHost = strings.TrimPrefix(origin, "http://")
		} else if strings.HasPrefix(origin, "https://") {
			originHost = strings.TrimPrefix(origin, "https://")
		}

		// Extract just the hostname/IP part (remove port)
		if colonIdx := strings.IndexByte(originHost, ':'); colonIdx != -1 {
			originHost = originHost[:colonIdx]
		}

		// SECURITY: Validate that the peer IP is also from a private network
		// This mitigates CSWSH where a malicious page on the same LAN tries to
		// hijack a WebSocket connection using the victim's session cookie
		peerIP := extractPeerIP(r.RemoteAddr)
		peerIsPrivate := isValidPrivateOrigin(peerIP)

		// Check if it's a valid private IP or localhost
		if isValidPrivateOrigin(originHost) {
			if !peerIsPrivate {
				// Origin claims to be private but peer is public - suspicious
				log.Warn().
					Str("origin", origin).
					Str("origin_host", originHost).
					Str("peer_ip", peerIP).
					Msg("WebSocket rejected - origin is private but peer is public (potential CSWSH)")
				return false
			}
			log.Debug().
				Str("origin", origin).
				Str("host", originHost).
				Str("peer_ip", peerIP).
				Msg("Allowing WebSocket connection from private network (no explicit origins configured)")
			return true
		}

		// Note: same-origin match already handled above (line 116)
		log.Warn().
			Str("origin", origin).
			Str("requestOrigin", requestOrigin).
			Str("peer_ip", peerIP).
			Msg("WebSocket connection rejected - not from allowed local/private network")
		return false
	}

	log.Warn().
		Str("origin", origin).
		Str("requestOrigin", requestOrigin).
		Strs("allowedOrigins", allowedOrigins).
		Msg("WebSocket connection rejected due to CORS")

	return false
}

// Client represents a WebSocket client
type Client struct {
	hub           *Hub
	conn          *websocket.Conn
	send          chan []byte
	sendMu        sync.RWMutex
	id            string
	orgID         string // Organization ID for tenant isolation
	lastPing      time.Time
	closed        atomic.Bool // Set when the client is unregistered; prevents sends to closed channel
	writeFailures int32       // Consecutive write failures; disconnects after maxWriteFailures
	lifecycleMu   sync.Mutex
	lifecycleDone chan struct{}
	lifecycleOnce sync.Once
	stateMu       sync.Mutex
	stateSnapshot *clientStateSnapshot
	// restRecovery is protected by stateMu. While set, REST supplies complete
	// display snapshots and the socket deliberately has no delta baseline.
	restRecovery    bool
	maxInboundBytes atomic.Int64
	// authorizeWrite is immutable after admission and called only by writePump.
	// Authentication remains owned by the API, not this transport package.
	authorizeWrite func() bool
}

func (c *Client) inboundLimitBytes() int64 {
	return c.maxInboundBytes.Load()
}

func (c *Client) applyAdvertisedInboundLimit(raw string) {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return
	}
	limit, err := strconv.ParseInt(raw, 10, 64)
	if err != nil || limit < minAdvertisedInboundBytes {
		log.Debug().Str("client", c.id).Str("value", raw).Msg("ignoring invalid client inbound frame limit")
		return
	}
	c.maxInboundBytes.Store(limit)
}

func (c *Client) lifecycleSignal() chan struct{} {
	c.lifecycleMu.Lock()
	defer c.lifecycleMu.Unlock()
	if c.lifecycleDone == nil {
		c.lifecycleDone = make(chan struct{})
	}
	return c.lifecycleDone
}

func (c *Client) initializeLifecycle() {
	c.lifecycleSignal()
}

func (c *Client) closeLifecycle() {
	c.lifecycleOnce.Do(func() {
		close(c.lifecycleSignal())
	})
}

func (c *Client) closeSend() {
	c.sendMu.Lock()
	defer c.sendMu.Unlock()
	if !c.closed.Swap(true) {
		close(c.send)
	}
}

// safeSend attempts to send data to the client's send channel.
// Returns false if the client is closed or the channel buffer is full.
func (c *Client) safeSend(data []byte) (sent bool) {
	c.sendMu.RLock()
	defer c.sendMu.RUnlock()

	if c.closed.Load() {
		return false
	}

	select {
	case c.send <- data:
		return true
	default:
		return false
	}
}

func (c *Client) queueFullState(messageType string, state interface{}) (int, bool, error) {
	snapshot, err := buildClientStateSnapshot(state)
	if err != nil {
		return 0, false, err
	}
	data, err := json.Marshal(Message{Type: messageType, Data: state})
	if err != nil {
		return 0, false, err
	}

	c.stateMu.Lock()
	defer c.stateMu.Unlock()
	if limit := c.inboundLimitBytes(); limit > 0 && int64(len(data)) > limit {
		return c.queueStateTooLargeLocked(messageType, snapshot, len(data), limit)
	}
	if !c.safeSend(data) {
		return len(data), false, nil
	}
	c.stateSnapshot = snapshot
	c.restRecovery = false
	return len(data), true, nil
}

// queueStateTooLargeLocked sends a compact REST-hydration marker without
// adopting the state as a delta baseline. REST builds its snapshot later and
// independently, so treating the withheld state as accepted would let a delta
// race onto a different baseline. The caller must hold stateMu.
func (c *Client) queueStateTooLargeLocked(
	supersedes string,
	snapshot *clientStateSnapshot,
	payloadBytes int,
	limit int64,
) (int, bool, error) {
	marker, err := json.Marshal(Message{
		Type: stateTooLargeMessageType,
		Data: stateTooLargePayload{
			Supersedes:    supersedes,
			Bytes:         payloadBytes,
			MaxBytes:      limit,
			ResourceCount: len(snapshot.resourceOrder),
			HydrateFrom:   "/api/state",
		},
	})
	if err != nil {
		return payloadBytes, false, err
	}

	// Invalidate first. Even if the marker cannot queue, no later delta may be
	// based on the payload the client did not receive.
	c.stateSnapshot = nil
	c.restRecovery = true
	if !c.safeSend(marker) {
		return payloadBytes, false, nil
	}
	return payloadBytes, true, nil
}

type preparedStateDelta struct {
	data []byte
	err  error
}

func prepareStateDelta(previous, current *clientStateSnapshot) preparedStateDelta {
	delta, err := buildClientStateDelta(previous, current)
	if err != nil {
		return preparedStateDelta{err: err}
	}
	if len(delta) == 0 {
		return preparedStateDelta{}
	}
	data, err := json.Marshal(Message{Type: "rawData", Data: delta})
	return preparedStateDelta{data: data, err: err}
}

// This map belongs to one serial dispatch, never the Hub or a tenant cache.
// Pointer identity identifies an actually accepted immutable snapshot, not a
// hash or resource ID. A reconnect/re-hydration gets its own baseline.
type stateDeltaDispatch map[*clientStateSnapshot]preparedStateDelta

func (c *Client) queueStateDelta(current *clientStateSnapshot) (int, bool, bool, error) {
	return c.queueStateDeltaForDispatch(current, nil)
}

func (c *Client) queueStateDeltaForDispatch(current *clientStateSnapshot, dispatch stateDeltaDispatch) (int, bool, bool, error) {

	if current == nil {
		return 0, false, false, fmt.Errorf("current state snapshot is nil")
	}

	c.stateMu.Lock()
	defer c.stateMu.Unlock()
	if c.restRecovery {
		dataLen, sent, markerErr := c.queueStateTooLargeLocked(
			"rawData",
			current,
			0,
			c.inboundLimitBytes(),
		)
		return dataLen, true, sent, markerErr
	}
	if c.stateSnapshot == nil {
		// Initial hydration owns the baseline. A client still waiting for that
		// payload must not receive a delta based on state it has never seen.
		return 0, false, false, nil
	}

	prepared, found := dispatch[c.stateSnapshot]
	if !found {
		prepared = prepareStateDelta(c.stateSnapshot, current)
		if dispatch != nil {
			dispatch[c.stateSnapshot] = prepared
		}
	}
	if prepared.err != nil {
		return 0, true, false, prepared.err
	}
	if len(prepared.data) == 0 {
		c.stateSnapshot = current
		return 0, false, true, nil
	}
	data := prepared.data
	if limit := c.inboundLimitBytes(); limit > 0 && int64(len(data)) > limit {
		dataLen, sent, markerErr := c.queueStateTooLargeLocked("rawData", current, len(data), limit)
		return dataLen, true, sent, markerErr
	}
	if !c.safeSend(data) {
		return len(data), true, false, nil
	}
	c.stateSnapshot = current
	return len(data), true, true, nil
}

// cloneAlertData returns a broadcast-safe copy of alert data to avoid data races when
// downstream sanitization/encoding happens concurrently with alert manager mutations.
func cloneAlertData(alert interface{}) interface{} {
	switch a := alert.(type) {
	case *alerts.Alert:
		cloned := cloneAlert(a)
		return cloned
	case alerts.Alert:
		cloned := cloneAlert(&a)
		return cloned
	default:
		return alert
	}
}

// cloneAlert performs a deep copy of the mutable fields within alerts.Alert.
func cloneAlert(src *alerts.Alert) alerts.Alert {
	if src == nil {
		return alerts.Alert{}
	}
	clone := *src

	if src.AckTime != nil {
		t := *src.AckTime
		clone.AckTime = &t
	}

	if len(src.EscalationTimes) > 0 {
		clone.EscalationTimes = append([]time.Time(nil), src.EscalationTimes...)
	}

	if src.Metadata != nil {
		clone.Metadata = cloneMetadata(src.Metadata)
	}

	return clone
}

// cloneMetadata creates a deep copy of alert metadata to detach from shared maps/slices.
func cloneMetadata(src map[string]interface{}) map[string]interface{} {
	if src == nil {
		return nil
	}

	dst := make(map[string]interface{}, len(src))
	for k, v := range src {
		dst[k] = cloneMetadataValue(v)
	}
	return dst
}

func cloneMetadataValue(value interface{}) interface{} {
	switch v := value.(type) {
	case map[string]interface{}:
		return cloneMetadata(v)
	case map[string]string:
		m := make(map[string]interface{}, len(v))
		for key, val := range v {
			m[key] = val
		}
		return m
	case []interface{}:
		arr := make([]interface{}, len(v))
		for i, elem := range v {
			arr[i] = cloneMetadataValue(elem)
		}
		return arr
	case []string:
		arr := make([]string, len(v))
		copy(arr, v)
		return arr
	case []int:
		arr := make([]int, len(v))
		copy(arr, v)
		return arr
	case []float64:
		arr := make([]float64, len(v))
		copy(arr, v)
		return arr
	default:
		return v
	}
}

// TenantBroadcast represents a broadcast message targeted at a specific tenant.
type TenantBroadcast struct {
	OrgID   string
	Message Message
}

type stateBroadcastRequest struct {
	orgID string
}

// OrgAuthChecker checks if a user/token can access an organization.
type OrgAuthChecker interface {
	// CanAccessOrg checks if the given user (and optional token) can access the org.
	CanAccessOrg(userID string, token interface{}, orgID string) bool
}

// OrgAuthContextChecker is an OrgAuthChecker whose decision can depend on
// request-bound authority, such as a proxy-authenticated administrator.
type OrgAuthContextChecker interface {
	CanAccessOrgContext(ctx context.Context, userID string, token interface{}, orgID string) bool
}

// MultiTenantCheckResult contains the result of a multi-tenant check.
type MultiTenantCheckResult struct {
	Allowed        bool
	FeatureEnabled bool   // false = feature flag disabled
	Licensed       bool   // false = not licensed (only meaningful if FeatureEnabled is true)
	Reason         string // Human-readable reason for denial
}

// MultiTenantChecker checks if multi-tenant functionality is enabled and licensed.
type MultiTenantChecker interface {
	// CheckMultiTenant checks if multi-tenant is enabled (feature flag) and licensed for the org.
	// Returns a result with details about why access was denied if not allowed.
	CheckMultiTenant(ctx context.Context, orgID string) MultiTenantCheckResult
}

// Hub maintains active WebSocket clients and broadcasts messages
type Hub struct {
	clients            map[*Client]bool            // Flat client registry for lifecycle management
	clientsByTenant    map[string]map[*Client]bool // Per-tenant client tracking
	broadcast          chan []byte
	broadcastSeq       chan Message         // Sequenced broadcast channel for ordering
	tenantBroadcast    chan TenantBroadcast // Per-tenant broadcast channel
	register           chan *Client
	unregister         chan *Client
	stopChan           chan struct{} // Signals shutdown
	stopOnce           sync.Once
	runStarted         chan struct{}
	runDone            chan struct{}
	runStartOnce       sync.Once
	runDoneOnce        sync.Once
	initialStateWG     sync.WaitGroup
	stateBuildSlot     chan struct{}
	mu                 sync.RWMutex
	getState           func(orgID string) interface{} // Function to get state for specific tenant
	allowedOrigins     []string                       // Allowed origins for CORS
	orgAuthChecker     OrgAuthChecker                 // Org authorization checker
	multiTenantChecker MultiTenantChecker             // Multi-tenant feature flag and license checker
	isTrustedProxy     func(ip string) bool           // Optional: checks if peer IP is a trusted reverse proxy
	// Broadcast coalescing fields
	coalesceWindow     time.Duration
	coalescePending    *Message
	coalesceReady      *Message
	coalesceTimer      *time.Timer
	coalesceGeneration uint64
	nextGeneration     uint64
	coalesceMutex      sync.Mutex
	// Per-tenant coalescing
	tenantCoalescePending    map[string]*Message
	tenantCoalesceReady      map[string]*Message
	tenantCoalesceTimers     map[string]*time.Timer
	tenantCoalesceGeneration map[string]uint64
	stateBroadcastWake       chan struct{}
	stateBroadcastDone       chan struct{}
	// stateBroadcastInterval is the minimum spacing between state broadcasts
	// to one audience (all clients, or one tenant); zero disables spacing.
	stateBroadcastInterval   time.Duration
	lastStateBroadcast       time.Time
	tenantLastStateBroadcast map[string]time.Time
}

// defaultStateBroadcastInterval spaces state broadcasts to one audience. Each
// broadcast rebuilds and diffs the full frontend state, and every accepted
// agent report requests one, so a 100 ms debounce let broadcasts scale with
// agent count (#2199). Report-driven store refreshes run at most once per
// 2 seconds, so broadcasting more often mostly re-sent the same resources.
const defaultStateBroadcastInterval = 2 * time.Second

// Message represents a WebSocket message
type Message struct {
	Type      string      `json:"type"`
	Data      interface{} `json:"data"`
	Timestamp string      `json:"timestamp,omitempty"`
}

// SetStateGetter sets the tenant-aware state getter function.
func (h *Hub) SetStateGetter(getState func(orgID string) interface{}) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.getState = getState
}

// SetOrgAuthChecker sets the organization authorization checker.
func (h *Hub) SetOrgAuthChecker(checker OrgAuthChecker) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.orgAuthChecker = checker
}

// SetMultiTenantChecker sets the multi-tenant feature flag and license checker.
func (h *Hub) SetMultiTenantChecker(checker MultiTenantChecker) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.multiTenantChecker = checker
}

// SetTrustedProxyChecker sets the function used to verify whether a peer IP is a
// trusted reverse proxy. When set, X-Forwarded-Host/X-Forwarded-Proto are only
// trusted in checkOrigin when the peer passes this check.
func (h *Hub) SetTrustedProxyChecker(fn func(ip string) bool) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.isTrustedProxy = fn
}

func (h *Hub) hasStateGetter() bool {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.getState != nil
}

// getStateForClient returns the state for a specific client based on their tenant
func (h *Hub) getStateForClient(client *Client) interface{} {
	if client == nil {
		return h.getStateForOrg("")
	}
	return h.getStateForOrg(client.orgID)
}

func (h *Hub) getStateForOrg(orgID string) interface{} {
	h.mu.RLock()
	getState := h.getState
	h.mu.RUnlock()

	if getState != nil {
		return getState(normalizeOrgID(orgID))
	}

	return nil
}

// NewHub creates a new WebSocket hub
func NewHub(getState func(orgID string) interface{}) *Hub {
	return &Hub{
		clients:                  make(map[*Client]bool),
		clientsByTenant:          make(map[string]map[*Client]bool),
		broadcast:                make(chan []byte, 256),
		broadcastSeq:             make(chan Message, 256),         // Buffered sequenced channel
		tenantBroadcast:          make(chan TenantBroadcast, 256), // Per-tenant broadcasts
		register:                 make(chan *Client),
		unregister:               make(chan *Client),
		stopChan:                 make(chan struct{}),
		runStarted:               make(chan struct{}),
		runDone:                  make(chan struct{}),
		stateBuildSlot:           make(chan struct{}, 1),
		getState:                 getState,
		allowedOrigins:           []string{},             // Default to empty (will be set based on actual host)
		coalesceWindow:           100 * time.Millisecond, // Coalesce rapid updates within 100ms
		tenantCoalescePending:    make(map[string]*Message),
		tenantCoalesceReady:      make(map[string]*Message),
		tenantCoalesceTimers:     make(map[string]*time.Timer),
		tenantCoalesceGeneration: make(map[string]uint64),
		stateBroadcastWake:       make(chan struct{}, 1),
		stateBroadcastDone:       make(chan struct{}),
		stateBroadcastInterval:   defaultStateBroadcastInterval,
		tenantLastStateBroadcast: make(map[string]time.Time),
	}
}

// Run starts the hub's main loop
func (h *Hub) Run() {
	h.runStartOnce.Do(func() {
		close(h.runStarted)
	})

	// Start broadcast sequencer goroutine
	sequencerDone := make(chan struct{})
	go func() {
		defer close(sequencerDone)
		h.runBroadcastSequencer()
	}()
	go h.runStateBroadcastWorker()
	defer func() {
		<-sequencerDone
		<-h.stateBroadcastDone
		h.runDoneOnce.Do(func() {
			close(h.runDone)
		})
	}()
	log.Info().Msg("WebSocket state payload configured for unified resources")

	pingTicker := time.NewTicker(30 * time.Second)
	defer pingTicker.Stop()

	for {
		select {
		case client := <-h.register:
			client.initializeLifecycle()
			h.mu.Lock()
			h.clients[client] = true
			// Legacy clients belong to the default tenant.
			orgID := normalizeOrgID(client.orgID)
			if h.clientsByTenant[orgID] == nil {
				h.clientsByTenant[orgID] = make(map[*Client]bool)
			}
			h.clientsByTenant[orgID][client] = true
			h.mu.Unlock()
			log.Info().Str("client", client.id).Str("org_id", client.orgID).Msg("webSocket client connected")

			// Send initial state to the new client immediately
			// Use tenant-aware state getter if available
			hasGetState := h.hasStateGetter()
			log.Debug().Bool("hasGetState", hasGetState).Msg("Checking getState function for new client")
			if hasGetState {
				h.initialStateWG.Add(1)
				go func() {
					defer h.initialStateWG.Done()
					h.sendInitialState(client)
				}()
			} else {
				log.Warn().
					Str("component", websocketHubComponent).
					Str("action", "initial_state_skipped_missing_getter").
					Str("client", client.id).
					Str("org_id", client.orgID).
					Msg("No getState function defined")
			}

		case client := <-h.unregister:
			h.mu.Lock()
			if h.removeClientLocked(client) {
				h.mu.Unlock()
				log.Info().Str("client", client.id).Str("org_id", client.orgID).Msg("webSocket client disconnected")
			} else {
				h.mu.Unlock()
			}

		case message := <-h.broadcast:
			h.dispatchToClients(message, "Client send channel full, dropping broadcast message and closing connection")

		case <-pingTicker.C:
			h.sendPing()

		case <-h.stopChan:
			log.Info().Msg("webSocket hub shutting down")
			// Cancel delayed initial-state work first, then let every producer
			// finish before closing client channels. Broadcast workers and
			// initial-state delivery otherwise race a final safeSend against
			// channel closure during shutdown.
			h.mu.Lock()
			for client := range h.clients {
				client.closeLifecycle()
			}
			for _, tenantClients := range h.clientsByTenant {
				for client := range tenantClients {
					client.closeLifecycle()
				}
			}
			h.mu.Unlock()

			<-sequencerDone
			<-h.stateBroadcastDone
			h.initialStateWG.Wait()

			h.mu.Lock()
			for client := range h.clients {
				client.closeSend()
			}
			for _, tenantClients := range h.clientsByTenant {
				for client := range tenantClients {
					client.closeSend()
				}
			}
			h.clients = make(map[*Client]bool)
			h.clientsByTenant = make(map[string]map[*Client]bool)
			h.mu.Unlock()
			return
		}
	}
}

// Stop gracefully shuts down the hub
func (h *Hub) Stop() {
	h.stopOnce.Do(func() {
		close(h.stopChan)
	})
	select {
	case <-h.runStarted:
		<-h.runDone
	default:
	}
	h.initialStateWG.Wait()
}

func (h *Hub) isStopping() bool {
	select {
	case <-h.stopChan:
		return true
	default:
		return false
	}
}

func (h *Hub) waitForActiveClient(client *Client, delay time.Duration) bool {
	timer := time.NewTimer(delay)
	defer timer.Stop()
	lifecycleDone := client.lifecycleSignal()

	select {
	case <-timer.C:
		h.mu.RLock()
		_, registered := h.clients[client]
		h.mu.RUnlock()
		return registered && !client.closed.Load() && !h.isStopping()
	case <-lifecycleDone:
		return false
	case <-h.stopChan:
		return false
	}
}

func (h *Hub) sendInitialState(client *Client) {
	log.Debug().Str("client", client.id).Msg("starting initial state goroutine")
	if !h.waitForActiveClient(client, initialWelcomeDelay) {
		log.Debug().Str("client", client.id).Msg("client disconnected before welcome message")
		return
	}

	welcomeMsg := Message{
		Type: "welcome",
		Data: map[string]string{"message": "Connected to Pulse WebSocket", "orgId": client.orgID},
	}
	data, err := json.Marshal(welcomeMsg)
	if err != nil {
		log.Error().Err(err).Str("client", client.id).Msg("Failed to marshal welcome message")
		return
	}
	log.Info().Str("client", client.id).Msg("sending welcome message")
	if !client.safeSend(data) {
		log.Warn().Str("client", client.id).Msg("failed to send welcome message - client closed or buffer full")
		return
	}
	log.Info().Str("client", client.id).Msg("welcome message sent")

	if !h.waitForActiveClient(client, initialStateMessageDelay) {
		log.Debug().Str("client", client.id).Msg("client disconnected before initial state")
		return
	}
	if !h.acquireStateBuildSlot(client) {
		log.Debug().Str("client", client.id).Msg("client disconnected while waiting to build initial state")
		return
	}
	defer h.releaseStateBuildSlot()

	log.Debug().Str("client", client.id).Msg("about to get state")
	stateData := h.getStateForClient(client)
	log.Debug().Str("client", client.id).Interface("stateType", fmt.Sprintf("%T", stateData)).Msg("got state for initial message")
	stateData = h.prepareStateForBroadcast(stateData)

	h.mu.RLock()
	_, stillRegistered := h.clients[client]
	h.mu.RUnlock()
	if !stillRegistered {
		log.Debug().Str("client", client.id).Msg("client disconnected before initial state")
		return
	}

	dataLen, sent, err := client.queueFullState("initialState", stateData)
	if err != nil {
		log.Error().Err(err).Str("client", client.id).Msg("failed to marshal initial state")
		return
	}

	log.Info().Str("client", client.id).Int("dataLen", dataLen).Int("dataKB", dataLen/1024).Msg("sending initial state to client")
	if sent {
		log.Info().Str("client", client.id).Msg("initial state sent successfully")
	} else {
		log.Warn().Str("client", client.id).Msg("client closed or buffer full, skipping initial state")
	}
}

func (h *Hub) sendRequestedState(client *Client) {
	if client == nil || !h.hasStateGetter() {
		return
	}
	if !h.acquireStateBuildSlot(client) {
		log.Debug().Str("client", client.id).Msg("client disconnected while waiting to build requested state")
		return
	}
	defer h.releaseStateBuildSlot()

	_, sent, err := client.queueFullState(
		"rawData",
		h.prepareStateForBroadcast(h.getStateForClient(client)),
	)
	if err != nil {
		log.Error().Err(err).Str("client", client.id).Msg("failed to marshal state for requestData")
		return
	}
	if !sent {
		log.Warn().Str("client", client.id).Msg("Failed to queue requestData state response; client channel closed or full")
	}
}

func (h *Hub) acquireStateBuildSlot(client *Client) bool {
	if client == nil {
		select {
		case h.stateBuildSlot <- struct{}{}:
			return true
		case <-h.stopChan:
			return false
		}
	}

	lifecycleDone := client.lifecycleSignal()
	select {
	case h.stateBuildSlot <- struct{}{}:
		h.mu.RLock()
		_, registered := h.clients[client]
		h.mu.RUnlock()
		if registered && !client.closed.Load() && !h.isStopping() {
			return true
		}
		h.releaseStateBuildSlot()
		return false
	case <-lifecycleDone:
		return false
	case <-h.stopChan:
		return false
	}
}

func (h *Hub) releaseStateBuildSlot() {
	<-h.stateBuildSlot
}

func (h *Hub) tryRegisterClient(client *Client) bool {
	if h.isStopping() {
		return false
	}

	select {
	case h.register <- client:
		return true
	case <-h.stopChan:
		return false
	}
}

// HandleWebSocket handles WebSocket upgrade requests
func (h *Hub) HandleWebSocket(w http.ResponseWriter, r *http.Request) {
	h.HandleWebSocketWithAuthorization(w, r, nil)
}

// HandleWebSocketWithAuthorization keeps an admitted connection subject to
// current authority. The callback runs only on its writer, before every frame
// (including queued frames and idle pings). Nil preserves existing callers.
func (h *Hub) HandleWebSocketWithAuthorization(w http.ResponseWriter, r *http.Request, authorize func() bool) {
	log.Info().
		Str("origin", r.Header.Get("Origin")).
		Str("host", r.Host).
		Str("userAgent", r.Header.Get("User-Agent")).
		Msg("WebSocket upgrade request")

	// Extract org ID from request for tenant isolation
	// Priority: Header > Cookie > Query param > Default
	orgID := strings.TrimSpace(r.Header.Get("X-Pulse-Org-ID"))
	if orgID == "" {
		if cookie, err := r.Cookie("pulse_org_id"); err == nil {
			orgID = strings.TrimSpace(cookie.Value)
		}
	}
	if orgID == "" {
		orgID = strings.TrimSpace(r.URL.Query().Get("org_id"))
	}
	if !isValidWebSocketOrgID(orgID) {
		log.Warn().
			Int("org_id_len", len(orgID)).
			Msg("WebSocket connection denied - invalid organization ID")
		http.Error(w, "Invalid organization ID", http.StatusBadRequest)
		return
	}

	// Multi-tenant feature flag and license check for non-default orgs
	h.mu.RLock()
	mtChecker := h.multiTenantChecker
	authChecker := h.orgAuthChecker
	h.mu.RUnlock()

	if orgID != "default" {
		// Check if multi-tenant is enabled and licensed
		if mtChecker != nil {
			result := mtChecker.CheckMultiTenant(r.Context(), orgID)
			if !result.Allowed {
				userID := getUserFromContext(r.Context())
				audit.Log(
					"websocket_multitenant_access_denied",
					userID,
					extractPeerIP(r.RemoteAddr),
					r.URL.Path,
					false,
					fmt.Sprintf("org_id=%s reason=%s feature_enabled=%t licensed=%t", orgID, result.Reason, result.FeatureEnabled, result.Licensed),
				)

				log.Warn().
					Str("org_id", orgID).
					Bool("feature_enabled", result.FeatureEnabled).
					Bool("licensed", result.Licensed).
					Str("reason", result.Reason).
					Msg("WebSocket connection denied - multi-tenant check failed")

				if !result.FeatureEnabled {
					// Feature flag disabled - 501 Not Implemented
					http.Error(w, "Multi-tenant functionality is not enabled", http.StatusNotImplemented)
				} else {
					// Feature enabled but not licensed - 402 Payment Required
					http.Error(w, "Multi-tenant access requires an Enterprise license", http.StatusPaymentRequired)
				}
				return
			}
		}
	}

	// Authorization check - auth context should already be set by
	// AuthContextMiddleware. Non-default organizations always require it. The
	// default organization is checked whenever a principal is present: in
	// multi-organization mode it is not open to every signed-in user, while an
	// installation without authentication has no principal to check.
	userID := getUserFromContext(r.Context())
	token := getAPITokenFromContext(r.Context())
	if authChecker != nil && (orgID != "default" || userID != "" || token != nil) {
		allowed := false
		if contextChecker, ok := authChecker.(OrgAuthContextChecker); ok {
			allowed = contextChecker.CanAccessOrgContext(r.Context(), userID, token, orgID)
		} else {
			allowed = authChecker.CanAccessOrg(userID, token, orgID)
		}
		if !allowed {
			audit.Log(
				"websocket_org_access_denied",
				userID,
				extractPeerIP(r.RemoteAddr),
				r.URL.Path,
				false,
				fmt.Sprintf("org_id=%s", orgID),
			)

			log.Warn().
				Str("org_id", orgID).
				Str("user_id", userID).
				Msg("WebSocket connection denied - unauthorized org access")
			http.Error(w, "Unauthorized to access organization", http.StatusForbidden)
			return
		}
	}

	// Create upgrader with our origin check.
	// Buffers are 64 KB: gorilla streams larger payloads across the buffer
	// automatically, so the previous 4 MB allocation per connection was
	// unnecessary overhead that scaled badly with concurrent clients
	// (100 clients × 8 MB = 800 MB just in buffers).
	upgrader := websocket.Upgrader{
		ReadBufferSize:    64 * 1024,
		WriteBufferSize:   64 * 1024,
		CheckOrigin:       h.checkOrigin,
		EnableCompression: true,
	}

	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Error().Err(err).Msg("failed to upgrade WebSocket connection")
		return
	}
	conn.EnableWriteCompression(true)
	if err := conn.SetCompressionLevel(flate.BestSpeed); err != nil {
		log.Warn().Err(err).Msg("Failed to set WebSocket compression level")
	}

	clientID := utils.GenerateID("client")
	client := &Client{
		hub:   h,
		conn:  conn,
		orgID: orgID,
		// Keep buffer bounded to avoid holding large state snapshots indefinitely if a client stalls.
		send:           make(chan []byte, 128),
		id:             clientID,
		lastPing:       time.Now(),
		authorizeWrite: authorize,
	}
	// This must be captured from the upgrade URL before registration: initial
	// state construction begins as soon as the hub accepts the client.
	client.applyAdvertisedInboundLimit(r.URL.Query().Get(inboundLimitQueryParam))

	log.Info().Str("client", clientID).Str("org_id", orgID).Msg("webSocket client created")

	if !client.hub.tryRegisterClient(client) {
		log.Info().Str("client", clientID).Str("org_id", orgID).Msg("WebSocket hub stopping; rejecting new client")
		conn.Close()
		return
	}

	// Start goroutines for reading and writing
	go client.writePump()
	go client.readPump()
}

// getUserFromContext extracts the user ID from request context.
func getUserFromContext(ctx context.Context) string {
	return auth.GetUser(ctx)
}

// getAPITokenFromContext extracts the API token from request context.
func getAPITokenFromContext(ctx context.Context) interface{} {
	return auth.GetAPIToken(ctx)
}

func normalizeOrgID(orgID string) string {
	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		return "default"
	}
	return orgID
}

// removeClientLocked removes a client from every hub index and closes the send
// channel once. h.mu must be held by the caller.
func (h *Hub) removeClientLocked(client *Client) bool {
	if client == nil {
		return false
	}

	removed := false
	if _, ok := h.clients[client]; ok {
		delete(h.clients, client)
		removed = true
	}

	if client.orgID != "" {
		if tenantClients := h.clientsByTenant[client.orgID]; tenantClients != nil {
			if _, ok := tenantClients[client]; ok {
				delete(tenantClients, client)
				removed = true
				if len(tenantClients) == 0 {
					delete(h.clientsByTenant, client.orgID)
				}
			}
		}
	} else {
		for orgID, tenantClients := range h.clientsByTenant {
			if _, ok := tenantClients[client]; ok {
				delete(tenantClients, client)
				removed = true
				if len(tenantClients) == 0 {
					delete(h.clientsByTenant, orgID)
				}
			}
		}
	}

	if removed {
		client.closeLifecycle()
		client.closeSend()
	}

	return removed
}

// dispatchToClients fan-outs a marshaled payload to all clients, dropping any that
// cannot keep up to prevent unbounded buffering.
func (h *Hub) dispatchToClients(data []byte, dropLog string) {
	h.mu.RLock()
	clients := make([]*Client, 0, len(h.clients))
	for client := range h.clients {
		clients = append(clients, client)
	}
	h.mu.RUnlock()

	for _, client := range clients {
		if !client.safeSend(data) {
			h.mu.Lock()
			if h.removeClientLocked(client) {
				log.Warn().Str("client", client.id).Msg(dropLog)
			}
			h.mu.Unlock()
		}
	}
}

func (h *Hub) markGlobalStateBroadcastReady(generation uint64) {
	h.coalesceMutex.Lock()
	if generation != h.coalesceGeneration || h.coalescePending == nil {
		h.coalesceMutex.Unlock()
		return
	}
	h.coalesceReady = h.coalescePending
	h.coalescePending = nil
	h.coalesceTimer = nil
	h.lastStateBroadcast = time.Now()
	h.coalesceMutex.Unlock()
	h.wakeStateBroadcastWorker()
}

func (h *Hub) markTenantStateBroadcastReady(orgID string, generation uint64) {
	h.coalesceMutex.Lock()
	if generation != h.tenantCoalesceGeneration[orgID] {
		h.coalesceMutex.Unlock()
		return
	}
	pending := h.tenantCoalescePending[orgID]
	if pending == nil {
		h.coalesceMutex.Unlock()
		return
	}
	h.tenantCoalesceReady[orgID] = pending
	delete(h.tenantCoalescePending, orgID)
	delete(h.tenantCoalesceTimers, orgID)
	delete(h.tenantCoalesceGeneration, orgID)
	h.tenantLastStateBroadcast[orgID] = time.Now()
	h.coalesceMutex.Unlock()
	h.wakeStateBroadcastWorker()
}

func (h *Hub) wakeStateBroadcastWorker() {
	select {
	case h.stateBroadcastWake <- struct{}{}:
	default:
	}
}

func (h *Hub) popReadyStateBroadcasts() (*Message, map[string]*Message) {
	h.coalesceMutex.Lock()
	defer h.coalesceMutex.Unlock()

	global := h.coalesceReady
	h.coalesceReady = nil
	tenants := h.tenantCoalesceReady
	h.tenantCoalesceReady = make(map[string]*Message)
	return global, tenants
}

func (h *Hub) dispatchStateBroadcast(pending *Message, orgID string) {
	if pending == nil || h.isStopping() || !h.hasRecipientsForMessage(*pending, orgID) {
		return
	}
	if !h.acquireStateBuildSlot(nil) {
		return
	}
	defer h.releaseStateBuildSlot()
	if _, currentStateRequest := h.messageHasCurrentStateRequest(*pending); currentStateRequest {
		prepared, ok := h.prepareMessageForMarshal(*pending, orgID)
		if !ok {
			return
		}
		h.dispatchCurrentStateSnapshot(prepared.Data, orgID)
		return
	}
	data, ok := h.marshalBroadcastMessage(*pending, orgID)
	if !ok {
		return
	}
	if orgID == "" {
		h.dispatchToClients(data, "Client send channel full, dropping coalesced message and closing connection")
		return
	}
	h.dispatchToTenantClients(orgID, data, "Client send channel full, dropping tenant coalesced message and closing connection")
}

func (h *Hub) dispatchCurrentStateSnapshot(state interface{}, orgID string) {
	snapshot, err := buildClientStateSnapshot(state)
	if err != nil {
		log.Error().Err(err).Str("org_id", normalizeOrgID(orgID)).Msg("Failed to build WebSocket state delta snapshot")
		return
	}

	h.mu.RLock()
	clients := make([]*Client, 0, len(h.clients))
	if orgID == "" {
		for client := range h.clients {
			clients = append(clients, client)
		}
	} else if tenantClients := h.clientsByTenant[orgID]; tenantClients != nil {
		for client := range tenantClients {
			clients = append(clients, client)
		}
	}
	h.mu.RUnlock()

	dispatch := make(stateDeltaDispatch)
	for _, client := range clients {
		dataLen, attempted, sent, queueErr := client.queueStateDeltaForDispatch(snapshot, dispatch)
		if queueErr != nil {
			log.Error().Err(queueErr).Str("client", client.id).Str("org_id", client.orgID).Msg("Failed to marshal WebSocket state delta")
			continue
		}
		if !attempted || sent {
			if sent {
				log.Debug().Str("client", client.id).Int("dataLen", dataLen).Msg("queued WebSocket state delta")
			}
			continue
		}
		h.mu.Lock()
		if h.removeClientLocked(client) {
			log.Warn().Str("client", client.id).Str("org_id", client.orgID).Msg("Client send channel full, dropping state delta and closing connection")
		}
		h.mu.Unlock()
	}
}

func (h *Hub) runStateBroadcastWorker() {
	defer close(h.stateBroadcastDone)
	for {
		select {
		case <-h.stateBroadcastWake:
			global, tenants := h.popReadyStateBroadcasts()
			h.dispatchStateBroadcast(global, "")
			for orgID, pending := range tenants {
				h.dispatchStateBroadcast(pending, orgID)
			}
		case <-h.stopChan:
			return
		}
	}
}

func (h *Hub) messageHasCurrentStateRequest(msg Message) (stateBroadcastRequest, bool) {
	req, ok := msg.Data.(stateBroadcastRequest)
	return req, ok
}

func (h *Hub) hasRecipientsForMessage(msg Message, orgID string) bool {
	if _, ok := h.messageHasCurrentStateRequest(msg); !ok {
		return true
	}
	if orgID != "" {
		return h.GetTenantClientCount(orgID) > 0
	}
	return h.GetClientCount() > 0
}

func (h *Hub) prepareMessageForMarshal(msg Message, orgID string) (Message, bool) {
	req, ok := h.messageHasCurrentStateRequest(msg)
	if !ok {
		return msg, true
	}

	requestOrgID := strings.TrimSpace(req.orgID)
	if requestOrgID == "" {
		requestOrgID = orgID
	}

	h.mu.RLock()
	hasGetter := h.getState != nil
	h.mu.RUnlock()
	if !hasGetter {
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "current_state_broadcast_skipped_missing_getter").
			Str("org_id", normalizeOrgID(requestOrgID)).
			Msg("Skipping current state broadcast because no state getter is configured")
		return Message{}, false
	}

	msg.Data = h.prepareStateForBroadcast(h.getStateForOrg(requestOrgID))
	return msg, true
}

func (h *Hub) marshalBroadcastMessage(msg Message, orgID string) ([]byte, bool) {
	prepared, ok := h.prepareMessageForMarshal(msg, orgID)
	if !ok {
		return nil, false
	}
	data, err := json.Marshal(prepared)
	if err != nil {
		if orgID != "" {
			log.Error().Err(err).Str("org_id", orgID).Str("type", prepared.Type).Msg("Failed to marshal tenant broadcast message")
		} else {
			log.Error().Err(err).Str("type", prepared.Type).Msg("Failed to marshal broadcast message")
		}
		return nil, false
	}
	return data, true
}

// stateBroadcastDelayLocked returns how long a newly requested state
// broadcast waits: coalesceWindow to merge simultaneous signals, and at least
// stateBroadcastInterval after the audience's previous broadcast. The caller
// holds coalesceMutex.
func (h *Hub) stateBroadcastDelayLocked(last time.Time) time.Duration {
	delay := h.coalesceWindow
	if !last.IsZero() && h.stateBroadcastInterval > 0 {
		if wait := h.stateBroadcastInterval - time.Since(last); wait > delay {
			delay = wait
		}
	}
	return delay
}

// runBroadcastSequencer handles sequenced broadcasts with coalescing for rapid state updates
func (h *Hub) runBroadcastSequencer() {
	for {
		select {
		case msg := <-h.broadcastSeq:
			// Handle raw data (state) messages with coalescing
			if msg.Type == "rawData" {
				h.coalesceMutex.Lock()

				// The latest state wins. A due broadcast is not rescheduled, so
				// a steady stream of signals cannot postpone it.
				current := msg
				h.coalescePending = &current
				if h.coalesceTimer == nil {
					h.nextGeneration++
					generation := h.nextGeneration
					h.coalesceGeneration = generation
					h.coalesceTimer = time.AfterFunc(h.stateBroadcastDelayLocked(h.lastStateBroadcast), func() {
						h.markGlobalStateBroadcastReady(generation)
					})
				}

				h.coalesceMutex.Unlock()
			} else {
				// Non-state messages (alerts, etc.) - send immediately
				if data, ok := h.marshalBroadcastMessage(msg, ""); ok {
					h.dispatchToClients(data, "Client send channel full, dropping message and closing connection")
				}
			}

		case tb := <-h.tenantBroadcast:
			// Handle tenant-specific broadcasts with coalescing
			if tb.Message.Type == "rawData" {
				h.coalesceMutex.Lock()

				// The latest state wins; a due tenant broadcast keeps its time.
				msgCopy := tb.Message
				h.tenantCoalescePending[tb.OrgID] = &msgCopy
				if h.tenantCoalesceTimers[tb.OrgID] == nil {
					h.nextGeneration++
					generation := h.nextGeneration
					h.tenantCoalesceGeneration[tb.OrgID] = generation
					orgID := tb.OrgID // Capture for closure
					h.tenantCoalesceTimers[orgID] = time.AfterFunc(h.stateBroadcastDelayLocked(h.tenantLastStateBroadcast[orgID]), func() {
						h.markTenantStateBroadcastReady(orgID, generation)
					})
				}

				h.coalesceMutex.Unlock()
			} else {
				// Non-state messages - send immediately to tenant
				if data, ok := h.marshalBroadcastMessage(tb.Message, tb.OrgID); ok {
					h.dispatchToTenantClients(tb.OrgID, data, "Client send channel full, dropping tenant message and closing connection")
				}
			}

		case <-h.stopChan:
			log.Debug().Msg("broadcast sequencer shutting down")
			// Cancel pending timer if exists
			h.coalesceMutex.Lock()
			if h.coalesceTimer != nil {
				h.coalesceTimer.Stop()
			}
			// Cancel all tenant timers
			for _, timer := range h.tenantCoalesceTimers {
				if timer != nil {
					timer.Stop()
				}
			}
			h.coalesceMutex.Unlock()
			return
		}
	}
}

// BroadcastState broadcasts state update to all clients via sequencer
func (h *Hub) BroadcastState(state interface{}) {
	if h.isStopping() {
		log.Debug().Msg("Skipping state broadcast while hub is stopping")
		return
	}

	log.Debug().Msg("broadcasting state")
	stateData := h.prepareStateForBroadcast(state)

	msg := Message{
		Type: "rawData",
		Data: stateData,
	}

	// Send through sequencer for ordering and coalescing
	select {
	case h.broadcastSeq <- msg:
	default:
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "enqueue_state_dropped").
			Str("channel", "broadcast_seq").
			Int("channel_depth", len(h.broadcastSeq)).
			Int("channel_capacity", cap(h.broadcastSeq)).
			Msg("Broadcast sequencer channel full, dropping state update")
	}
}

// BroadcastCurrentState coalesces a state-change signal and resolves the current
// tenant-aware state only when the broadcast window is ready to flush.
func (h *Hub) BroadcastCurrentState() {
	if h.isStopping() {
		log.Debug().Msg("Skipping current state broadcast while hub is stopping")
		return
	}

	log.Debug().Msg("broadcasting current state")
	msg := Message{
		Type: "rawData",
		Data: stateBroadcastRequest{},
	}

	select {
	case h.broadcastSeq <- msg:
	default:
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "enqueue_state_dropped").
			Str("channel", "broadcast_seq").
			Int("channel_depth", len(h.broadcastSeq)).
			Int("channel_capacity", cap(h.broadcastSeq)).
			Msg("Broadcast sequencer channel full, dropping state update")
	}
}

// BroadcastStateToTenant broadcasts state update only to clients of a specific tenant.
func (h *Hub) BroadcastStateToTenant(orgID string, state interface{}) {
	if h.isStopping() {
		log.Debug().Str("org_id", orgID).Msg("Skipping tenant state broadcast while hub is stopping")
		return
	}

	log.Debug().Str("org_id", orgID).Msg("Broadcasting state to tenant")
	stateData := h.prepareStateForBroadcast(state)

	msg := Message{
		Type: "rawData",
		Data: stateData,
	}

	// Send through tenant broadcast channel
	select {
	case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
	default:
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "enqueue_tenant_state_dropped").
			Str("org_id", orgID).
			Str("channel", "tenant_broadcast").
			Int("channel_depth", len(h.tenantBroadcast)).
			Int("channel_capacity", cap(h.tenantBroadcast)).
			Msg("Tenant broadcast channel full, dropping state update")
	}
}

// BroadcastCurrentStateToTenant coalesces a tenant state-change signal and
// resolves that tenant's current state only when the broadcast window flushes.
func (h *Hub) BroadcastCurrentStateToTenant(orgID string) {
	orgID = normalizeOrgID(orgID)
	if h.isStopping() {
		log.Debug().Str("org_id", orgID).Msg("Skipping tenant current state broadcast while hub is stopping")
		return
	}

	log.Debug().Str("org_id", orgID).Msg("Broadcasting current state to tenant")
	msg := Message{
		Type: "rawData",
		Data: stateBroadcastRequest{orgID: orgID},
	}

	select {
	case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
	default:
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "enqueue_tenant_state_dropped").
			Str("org_id", orgID).
			Str("channel", "tenant_broadcast").
			Int("channel_depth", len(h.tenantBroadcast)).
			Int("channel_capacity", cap(h.tenantBroadcast)).
			Msg("Tenant broadcast channel full, dropping state update")
	}
}

// dispatchToTenantClients sends a message only to clients of a specific tenant.
func (h *Hub) dispatchToTenantClients(orgID string, data []byte, dropLog string) {
	h.mu.RLock()
	tenantClients := h.clientsByTenant[orgID]
	if tenantClients == nil {
		h.mu.RUnlock()
		return
	}
	clients := make([]*Client, 0, len(tenantClients))
	for client := range tenantClients {
		clients = append(clients, client)
	}
	h.mu.RUnlock()

	for _, client := range clients {
		if !client.safeSend(data) {
			h.mu.Lock()
			if h.removeClientLocked(client) {
				log.Warn().Str("client", client.id).Str("org_id", client.orgID).Msg(dropLog)
			}
			h.mu.Unlock()
		}
	}
}

// prepareStateForBroadcast applies websocket payload compatibility rules to state payloads.
func (h *Hub) prepareStateForBroadcast(state interface{}) interface{} {
	return state
}

// GetTenantClientCount returns the number of connected clients for a specific tenant.
func (h *Hub) GetTenantClientCount(orgID string) int {
	orgID = normalizeOrgID(orgID)
	h.mu.RLock()
	defer h.mu.RUnlock()
	if tenantClients := h.clientsByTenant[orgID]; tenantClients != nil {
		return len(tenantClients)
	}
	return 0
}

// BroadcastAlertToTenant broadcasts alert to clients of a specific tenant only.
// Empty org IDs are normalized to the default tenant.
func (h *Hub) BroadcastAlertToTenant(orgID string, alert interface{}) {
	orgID = normalizeOrgID(orgID)

	log.Info().Str("org_id", orgID).Msg("broadcasting alert to tenant WebSocket clients")
	msg := Message{
		Type: "alert",
		Data: cloneAlertData(alert),
	}
	if h.isStopping() {
		log.Debug().Str("org_id", orgID).Msg("Skipping tenant alert broadcast while hub is stopping")
		return
	}

	select {
	case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
	default:
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "enqueue_tenant_alert_dropped").
			Str("org_id", orgID).
			Str("channel", "tenant_broadcast").
			Int("channel_depth", len(h.tenantBroadcast)).
			Int("channel_capacity", cap(h.tenantBroadcast)).
			Msg("Tenant broadcast channel full, dropping alert")
	}
}

// BroadcastAlertResolvedToTenant broadcasts alert resolution to clients of a specific tenant only.
// Empty org IDs are normalized to the default tenant.
func (h *Hub) BroadcastAlertResolvedToTenant(orgID string, alertID string) {
	orgID = normalizeOrgID(orgID)

	log.Info().Str("org_id", orgID).Str("alertID", alertID).Msg("broadcasting alert resolved to tenant WebSocket clients")
	msg := Message{
		Type: "alertResolved",
		Data: map[string]string{
			"alertIdentifier": alertID,
		},
	}
	if h.isStopping() {
		log.Debug().Str("org_id", orgID).Msg("Skipping tenant alert resolved broadcast while hub is stopping")
		return
	}

	select {
	case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
	default:
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "enqueue_tenant_alert_resolved_dropped").
			Str("org_id", orgID).
			Str("channel", "tenant_broadcast").
			Int("channel_depth", len(h.tenantBroadcast)).
			Int("channel_capacity", cap(h.tenantBroadcast)).
			Msg("Tenant broadcast channel full, dropping alert resolved")
	}
}

// GetClientCount returns the number of connected clients
func (h *Hub) GetClientCount() int {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return len(h.clients)
}

// BroadcastToTenant sends a custom message to its owning organization.
func (h *Hub) BroadcastToTenant(orgID string, data interface{}) {
	h.BroadcastMessageToTenant(orgID, Message{
		Type:      "custom",
		Data:      data,
		Timestamp: time.Now().Format(time.RFC3339),
	})
}

// BroadcastMessageToTenant delivers an event only to its owning organization.
// Empty organization IDs identify the default tenant, including single-tenant installs.
func (h *Hub) BroadcastMessageToTenant(orgID string, msg Message) {
	orgID = normalizeOrgID(orgID)
	if h.isStopping() {
		return
	}
	msg.Data = sanitizeData(msg.Data)
	select {
	case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
	default:
		log.Warn().Str("component", websocketHubComponent).
			Str("action", "enqueue_tenant_message_dropped").Str("org_id", orgID).
			Str("message_type", msg.Type).Str("channel", "tenant_broadcast").
			Int("channel_depth", len(h.tenantBroadcast)).
			Int("channel_capacity", cap(h.tenantBroadcast)).
			Msg("Tenant broadcast channel full, dropping event")
	}
}

// BroadcastMessage accepts only instance-wide events. Tenant-derived messages
// must use BroadcastMessageToTenant with the organization owning their data.
func (h *Hub) BroadcastMessage(msg Message) {
	switch msg.Type {
	case "ping", "settingsUpdate", "update:progress":
		// Heartbeats, shared system theme, and server software update progress
		// describe the instance rather than any organization's infrastructure.
	default:
		log.Error().Str("type", msg.Type).Msg("Refusing unscoped WebSocket event")
		return
	}
	if h.isStopping() {
		log.Debug().Str("type", msg.Type).Msg("Skipping websocket broadcast while hub is stopping")
		return
	}

	// Sanitize the message data to handle NaN values
	msg.Data = sanitizeData(msg.Data)

	data, err := json.Marshal(msg)
	if err != nil {
		log.Error().Err(err).Str("type", msg.Type).Msg("failed to marshal WebSocket message")
		// Try to marshal without data to see what's failing
		debugMsg := Message{Type: msg.Type, Data: "[error marshaling data]"}
		if debugData, debugErr := json.Marshal(debugMsg); debugErr == nil {
			log.Debug().Str("debugMsg", string(debugData)).Msg("debug message")
		}
		return
	}

	select {
	case h.broadcast <- data:
	default:
		log.Warn().
			Str("component", websocketHubComponent).
			Str("action", "enqueue_broadcast_dropped").
			Str("message_type", msg.Type).
			Str("channel", "broadcast").
			Int("channel_depth", len(h.broadcast)).
			Int("channel_capacity", cap(h.broadcast)).
			Msg("WebSocket broadcast channel full")
	}
}

// sendPing sends a ping message to all clients
func (h *Hub) sendPing() {
	msg := Message{
		Type: "ping",
		Data: map[string]int64{"timestamp": time.Now().Unix()},
	}
	h.BroadcastMessage(msg)
}

func isUnexpectedClientClose(err error) bool {
	return websocket.IsUnexpectedCloseError(
		err,
		websocket.CloseNormalClosure,
		websocket.CloseGoingAway,
		websocket.CloseAbnormalClosure,
	)
}

// Gorilla's read limit counts encoded frame bytes. Bound the decompressed
// reader as well, before buffering or JSON decoding, including messages whose
// sender has not finished the final frame.
func (c *Client) readInboundMessage() ([]byte, error) {
	_, reader, err := c.conn.NextReader()
	if err != nil {
		return nil, err
	}
	message, err := io.ReadAll(io.LimitReader(reader, maxWebSocketInboundMessageSize+1))
	if err != nil {
		return nil, err
	}
	if len(message) > maxWebSocketInboundMessageSize {
		if err := c.conn.WriteControl(websocket.CloseMessage,
			websocket.FormatCloseMessage(websocket.CloseMessageTooBig, ""),
			time.Now().Add(time.Second)); err != nil {
			log.Debug().Err(err).Str("client", c.id).Msg("Failed to send WebSocket size-limit close")
		}
		return nil, websocket.ErrReadLimit
	}
	return message, nil
}

// readPump handles incoming messages from the client
func (c *Client) readPump() {
	defer func() {
		log.Info().Str("client", c.id).Msg("readPump exiting")
		c.hub.unregister <- c
		if err := c.conn.Close(); err != nil {
			log.Debug().Err(err).Str("client", c.id).Msg("Failed to close WebSocket connection in readPump")
		}
	}()

	if err := c.conn.SetReadDeadline(time.Now().Add(clientReadDeadline)); err != nil {
		log.Warn().Err(err).Str("client", c.id).Msg("failed to set initial read deadline")
	}
	c.conn.SetReadLimit(maxWebSocketInboundMessageSize)
	c.conn.SetPongHandler(func(string) error {
		if err := c.conn.SetReadDeadline(time.Now().Add(clientReadDeadline)); err != nil {
			log.Warn().Err(err).Str("client", c.id).Msg("failed to refresh read deadline on pong")
		}
		c.lastPing = time.Now()
		return nil
	})

	log.Info().Str("client", c.id).Msg("readPump started")

	for {
		message, err := c.readInboundMessage()
		if err != nil {
			if isUnexpectedClientClose(err) {
				log.Error().Err(err).Str("client", c.id).Msg("webSocket read error")
			} else {
				log.Info().Err(err).Str("client", c.id).Msg("webSocket closed")
			}
			break
		}

		// Any successful read proves the peer is alive; without this refresh a
		// client whose pong frames are lost in transit dies at the deadline even
		// while its 30s JSON heartbeats are arriving.
		if err := c.conn.SetReadDeadline(time.Now().Add(clientReadDeadline)); err != nil {
			log.Warn().Err(err).Str("client", c.id).Msg("failed to refresh read deadline on message")
		}

		// Handle incoming messages
		var msg Message
		if err := json.Unmarshal(message, &msg); err != nil {
			log.Error().Err(err).Str("client", c.id).Msg("failed to unmarshal WebSocket message")
			continue
		}

		// Handle different message types
		switch msg.Type {
		case "ping":
			// Respond with pong
			pong := Message{
				Type: "pong",
				Data: map[string]int64{"timestamp": time.Now().Unix()},
			}
			if data, err := json.Marshal(pong); err == nil {
				if !c.safeSend(data) {
					log.Warn().Str("client", c.id).Msg("Failed to queue pong response; client channel closed or full")
				}
			} else {
				log.Error().Err(err).Str("client", c.id).Msg("Failed to marshal pong response")
			}
		case "requestData":
			c.hub.sendRequestedState(c)
		default:
			log.Debug().Str("client", c.id).Str("type", msg.Type).Msg("received WebSocket message")
		}
	}
}

// allowWrite checks current authority without disclosing authentication detail.
func (c *Client) allowWrite() bool {
	if c.authorizeWrite == nil || c.authorizeWrite() {
		return true
	}
	// Never send an HTTP error or queued data after authority ends. The fixed
	// policy close is understood by the existing frontend authentication path.
	_ = c.conn.WriteControl(websocket.CloseMessage,
		websocket.FormatCloseMessage(websocket.ClosePolicyViolation, "Monitoring access ended"),
		time.Now().Add(10*time.Second))
	return false
}

// writePump handles outgoing messages to the client.
func (c *Client) writePump() {
	// Maximum consecutive write failures before disconnecting.
	// This provides graceful degradation for slow clients.
	const maxWriteFailures = 3
	// Write deadline for messages. Increased from 10s to 30s to handle
	// large state payloads on slower connections (e.g., Raspberry Pi, slow networks).
	const writeDeadline = 30 * time.Second
	// Ping deadline can be shorter since pings are small
	const pingDeadline = 10 * time.Second

	ticker := time.NewTicker(clientPingPeriod)
	defer func() {
		log.Info().Str("client", c.id).Msg("writePump exiting")
		ticker.Stop()
		if err := c.conn.Close(); err != nil {
			log.Debug().Err(err).Str("client", c.id).Msg("Failed to close WebSocket connection in writePump")
		}
	}()

	log.Info().Str("client", c.id).Msg("writePump started")

	for {
		select {
		case message, ok := <-c.send:
			if err := c.conn.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil {
				log.Warn().Err(err).Str("client", c.id).Msg("failed to set write deadline before message send")
			}
			if !ok {
				log.Debug().Str("client", c.id).Msg("send channel closed")
				if err := c.conn.WriteMessage(websocket.CloseMessage, []byte{}); err != nil {
					log.Warn().Err(err).Str("client", c.id).Msg("failed to send close message")
				}
				return
			}
			if !c.allowWrite() {
				return
			}

			// Send the primary message
			if err := c.conn.WriteMessage(websocket.TextMessage, message); err != nil {
				c.writeFailures++
				log.Warn().
					Err(err).
					Str("client", c.id).
					Int("msgSize", len(message)).
					Int32("consecutiveFailures", c.writeFailures).
					Msg("Failed to write message")

				// Graceful degradation: only disconnect after multiple consecutive failures
				if c.writeFailures >= maxWriteFailures {
					log.Error().
						Str("client", c.id).
						Int32("failures", c.writeFailures).
						Msg("Too many consecutive write failures, disconnecting client")
					return
				}
				// Skip this message and continue - don't disconnect immediately
				continue
			}

			// Reset failure count on successful write
			c.writeFailures = 0

			// Send any queued messages
			n := len(c.send)
		flushLoop:
			for i := 0; i < n; i++ {
				select {
				case msg, ok := <-c.send:
					if !ok || !c.allowWrite() {
						return
					}
					if err := c.conn.WriteMessage(websocket.TextMessage, msg); err != nil {
						log.Warn().Err(err).Str("client", c.id).Int("msgSize", len(msg)).Msg("failed to flush queued message")
						// Don't disconnect on queued message failure, just break the flush loop
						break flushLoop
					}
				default:
					// No more messages
				}
			}

		case <-ticker.C:
			if !c.allowWrite() {
				return
			}
			if err := c.conn.SetWriteDeadline(time.Now().Add(pingDeadline)); err != nil {
				log.Warn().Err(err).Str("client", c.id).Msg("failed to set write deadline for ping")
			}
			if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
				log.Debug().Err(err).Str("client", c.id).Msg("failed to send ping; closing connection")
				return
			}
		}
	}
}

// sanitizeData recursively sanitizes data to replace NaN/Inf values with nil
func sanitizeData(data interface{}) interface{} {
	// First, marshal to JSON to convert structs to maps
	jsonBytes, err := json.Marshal(data)
	if err != nil {
		return data
	}

	var jsonData interface{}
	if err := json.Unmarshal(jsonBytes, &jsonData); err != nil {
		return data
	}

	return sanitizeValue(jsonData)
}

// sanitizeValue recursively sanitizes JSON-compatible values
func sanitizeValue(data interface{}) interface{} {
	switch v := data.(type) {
	case float64:
		if math.IsNaN(v) || math.IsInf(v, 0) {
			return nil
		}
		return v
	case float32:
		if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {
			return nil
		}
		return v
	case map[string]interface{}:
		sanitized := make(map[string]interface{})
		for k, val := range v {
			sanitized[k] = sanitizeValue(val)
		}
		return sanitized
	case []interface{}:
		sanitized := make([]interface{}, len(v))
		for i, val := range v {
			sanitized[i] = sanitizeValue(val)
		}
		return sanitized
	default:
		// For all other types (string, bool, nil, etc.), return as-is
		return v
	}
}