Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package websocket
import (
"compress/flate"
"context"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gorilla/websocket"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
"github.com/rcourtman/pulse-go-rewrite/pkg/audit"
"github.com/rcourtman/pulse-go-rewrite/pkg/auth"
"github.com/rs/zerolog/log"
)
const (
// maxWebSocketInboundMessageSize bounds both encoded and decoded inbound messages.
maxWebSocketInboundMessageSize = 64 * 1024
// maxWebSocketOrgIDLength keeps org IDs bounded to prevent oversized header/query abuse.
maxWebSocketOrgIDLength = 64
// inboundLimitQueryParam lets a browser declare the largest state frame it
// can safely accept before the hub builds that connection's initial state.
inboundLimitQueryParam = "max_message_bytes"
// minAdvertisedInboundBytes keeps tiny or malicious values from turning
// ordinary control messages into an unusable connection.
minAdvertisedInboundBytes = 64 * 1024
stateTooLargeMessageType = "stateTooLarge"
websocketHubComponent = "websocket_hub"
initialWelcomeDelay = 500 * time.Millisecond
initialStateMessageDelay = 100 * time.Millisecond
// clientReadDeadline is how long a client may stay silent before the server
// drops it. Liveness is proven by ANY inbound frame (data, JSON heartbeat,
// or protocol pong) — proxies and middleboxes (Cloudflare, AV products)
// sometimes eat ping/pong control frames, so pongs must not be the only
// accepted proof of life. Must be comfortably above clientPingPeriod and
// the frontend's 30s JSON heartbeat, including a background tab throttled
// to one heartbeat per minute.
clientReadDeadline = 90 * time.Second
// clientPingPeriod is how often writePump sends protocol pings. At 30s a
// pong-only client survives two consecutive lost ping/pong round trips
// within clientReadDeadline.
clientPingPeriod = 30 * time.Second
)
type stateTooLargePayload struct {
Supersedes string `json:"supersedes"`
Bytes int `json:"bytes,omitempty"`
MaxBytes int64 `json:"maxBytes"`
ResourceCount int `json:"resourceCount"`
HydrateFrom string `json:"hydrateFrom"`
}
// extractPeerIP extracts just the IP part from a RemoteAddr (host:port format)
func extractPeerIP(remoteAddr string) string {
host, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
// RemoteAddr might not have a port, try using it directly
return remoteAddr
}
return host
}
// isValidPrivateOrigin checks if the origin is from a valid private network
func isValidPrivateOrigin(host string) bool {
// Check localhost variations
if host == "localhost" || host == "127.0.0.1" || host == "::1" {
return true
}
// Check if it's a valid IP address
ip := net.ParseIP(host)
if ip != nil {
// Check if it's a private IP
return ip.IsLoopback() || ip.IsPrivate()
}
// Allow common local domain patterns but be more restrictive
// Only allow if it's clearly a local domain
if strings.HasSuffix(host, ".local") || strings.HasSuffix(host, ".lan") {
// But not arbitrary subdomains that could be malicious
parts := strings.Split(host, ".")
if len(parts) <= 3 { // hostname.local or hostname.subdomain.local
return true
}
}
return false
}
// normalizeForwardedProto coerces forwarded proto values into the HTTP scheme space so that
// websocket upgrades coming through proxies that emit ws/wss continue to compare equal to the
// browser-sent Origin header (which is always http/https).
func normalizeForwardedProto(proto string, fallback string) string {
if proto == "" {
return fallback
}
// Some proxies send comma-separated proto chains; take the first hop.
if comma := strings.IndexByte(proto, ','); comma != -1 {
proto = proto[:comma]
}
cleaned := strings.TrimSpace(strings.ToLower(proto))
switch cleaned {
case "wss":
return "https"
case "ws":
return "http"
case "https", "http":
return cleaned
default:
if cleaned != "" {
return cleaned
}
return fallback
}
}
func isValidWebSocketOrgID(orgID string) bool {
if orgID == "" || orgID == "." || orgID == ".." {
return false
}
if len(orgID) > maxWebSocketOrgIDLength {
return false
}
if strings.TrimSpace(orgID) != orgID {
return false
}
if strings.ContainsAny(orgID, `/\`) {
return false
}
for _, r := range orgID {
if r < 0x20 || r == 0x7f {
return false
}
}
return true
}
// SetAllowedOrigins sets the allowed origins for CORS
func (h *Hub) SetAllowedOrigins(origins []string) {
h.mu.Lock()
defer h.mu.Unlock()
h.allowedOrigins = append([]string(nil), origins...)
}
// checkOrigin validates the origin against allowed origins
func (h *Hub) checkOrigin(r *http.Request) bool {
origin := strings.TrimSpace(r.Header.Get("Origin"))
if origin == "" {
return false
}
h.mu.RLock()
allowedOrigins := append([]string(nil), h.allowedOrigins...)
h.mu.RUnlock()
// Determine the actual origin (accounting for proxy headers).
// Only trust X-Forwarded-* headers when the peer is a known trusted proxy,
// consistent with how auth.go gates proxy header trust via isTrustedProxyIP.
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
h.mu.RLock()
trustedProxyFn := h.isTrustedProxy
h.mu.RUnlock()
peerIP := extractPeerIP(r.RemoteAddr)
peerIsTrusted := trustedProxyFn != nil && trustedProxyFn(peerIP)
if peerIsTrusted {
if forwardedProto := r.Header.Get("X-Forwarded-Proto"); forwardedProto != "" {
scheme = normalizeForwardedProto(forwardedProto, scheme)
} else if forwardedScheme := r.Header.Get("X-Forwarded-Scheme"); forwardedScheme != "" {
scheme = normalizeForwardedProto(forwardedScheme, scheme)
}
}
host := r.Host
if peerIsTrusted {
if forwardedHost := r.Header.Get("X-Forwarded-Host"); forwardedHost != "" {
host = forwardedHost
}
}
requestOrigin := scheme + "://" + host
// Allow same-origin requests
if origin == requestOrigin {
return true
}
if securityutil.SameHostWebSocketOrigin(origin, host) {
return true
}
// Check if wildcard is allowed
for _, allowed := range allowedOrigins {
if allowed == "*" {
return true
}
if allowed == origin {
return true
}
}
// If no origins configured, only allow from truly private networks
// SECURITY: This is a relaxed policy for homelab deployments. For production,
// configure explicit allowed origins via WEBSOCKET_ALLOWED_ORIGINS env var.
if len(allowedOrigins) == 0 {
// Parse the origin URL to validate it properly
originHost := origin
if strings.HasPrefix(origin, "http://") {
originHost = strings.TrimPrefix(origin, "http://")
} else if strings.HasPrefix(origin, "https://") {
originHost = strings.TrimPrefix(origin, "https://")
}
// Extract just the hostname/IP part (remove port)
if colonIdx := strings.IndexByte(originHost, ':'); colonIdx != -1 {
originHost = originHost[:colonIdx]
}
// SECURITY: Validate that the peer IP is also from a private network
// This mitigates CSWSH where a malicious page on the same LAN tries to
// hijack a WebSocket connection using the victim's session cookie
peerIP := extractPeerIP(r.RemoteAddr)
peerIsPrivate := isValidPrivateOrigin(peerIP)
// Check if it's a valid private IP or localhost
if isValidPrivateOrigin(originHost) {
if !peerIsPrivate {
// Origin claims to be private but peer is public - suspicious
log.Warn().
Str("origin", origin).
Str("origin_host", originHost).
Str("peer_ip", peerIP).
Msg("WebSocket rejected - origin is private but peer is public (potential CSWSH)")
return false
}
log.Debug().
Str("origin", origin).
Str("host", originHost).
Str("peer_ip", peerIP).
Msg("Allowing WebSocket connection from private network (no explicit origins configured)")
return true
}
// Note: same-origin match already handled above (line 116)
log.Warn().
Str("origin", origin).
Str("requestOrigin", requestOrigin).
Str("peer_ip", peerIP).
Msg("WebSocket connection rejected - not from allowed local/private network")
return false
}
log.Warn().
Str("origin", origin).
Str("requestOrigin", requestOrigin).
Strs("allowedOrigins", allowedOrigins).
Msg("WebSocket connection rejected due to CORS")
return false
}
// Client represents a WebSocket client
type Client struct {
hub *Hub
conn *websocket.Conn
send chan []byte
sendMu sync.RWMutex
id string
orgID string // Organization ID for tenant isolation
lastPing time.Time
closed atomic.Bool // Set when the client is unregistered; prevents sends to closed channel
writeFailures int32 // Consecutive write failures; disconnects after maxWriteFailures
lifecycleMu sync.Mutex
lifecycleDone chan struct{}
lifecycleOnce sync.Once
stateMu sync.Mutex
stateSnapshot *clientStateSnapshot
// restRecovery is protected by stateMu. While set, REST supplies complete
// display snapshots and the socket deliberately has no delta baseline.
restRecovery bool
maxInboundBytes atomic.Int64
// authorizeWrite is immutable after admission and called only by writePump.
// Authentication remains owned by the API, not this transport package.
authorizeWrite func() bool
}
func (c *Client) inboundLimitBytes() int64 {
return c.maxInboundBytes.Load()
}
func (c *Client) applyAdvertisedInboundLimit(raw string) {
raw = strings.TrimSpace(raw)
if raw == "" {
return
}
limit, err := strconv.ParseInt(raw, 10, 64)
if err != nil || limit < minAdvertisedInboundBytes {
log.Debug().Str("client", c.id).Str("value", raw).Msg("ignoring invalid client inbound frame limit")
return
}
c.maxInboundBytes.Store(limit)
}
func (c *Client) lifecycleSignal() chan struct{} {
c.lifecycleMu.Lock()
defer c.lifecycleMu.Unlock()
if c.lifecycleDone == nil {
c.lifecycleDone = make(chan struct{})
}
return c.lifecycleDone
}
func (c *Client) initializeLifecycle() {
c.lifecycleSignal()
}
func (c *Client) closeLifecycle() {
c.lifecycleOnce.Do(func() {
close(c.lifecycleSignal())
})
}
func (c *Client) closeSend() {
c.sendMu.Lock()
defer c.sendMu.Unlock()
if !c.closed.Swap(true) {
close(c.send)
}
}
// safeSend attempts to send data to the client's send channel.
// Returns false if the client is closed or the channel buffer is full.
func (c *Client) safeSend(data []byte) (sent bool) {
c.sendMu.RLock()
defer c.sendMu.RUnlock()
if c.closed.Load() {
return false
}
select {
case c.send <- data:
return true
default:
return false
}
}
func (c *Client) queueFullState(messageType string, state interface{}) (int, bool, error) {
snapshot, err := buildClientStateSnapshot(state)
if err != nil {
return 0, false, err
}
data, err := json.Marshal(Message{Type: messageType, Data: state})
if err != nil {
return 0, false, err
}
c.stateMu.Lock()
defer c.stateMu.Unlock()
if limit := c.inboundLimitBytes(); limit > 0 && int64(len(data)) > limit {
return c.queueStateTooLargeLocked(messageType, snapshot, len(data), limit)
}
if !c.safeSend(data) {
return len(data), false, nil
}
c.stateSnapshot = snapshot
c.restRecovery = false
return len(data), true, nil
}
// queueStateTooLargeLocked sends a compact REST-hydration marker without
// adopting the state as a delta baseline. REST builds its snapshot later and
// independently, so treating the withheld state as accepted would let a delta
// race onto a different baseline. The caller must hold stateMu.
func (c *Client) queueStateTooLargeLocked(
supersedes string,
snapshot *clientStateSnapshot,
payloadBytes int,
limit int64,
) (int, bool, error) {
marker, err := json.Marshal(Message{
Type: stateTooLargeMessageType,
Data: stateTooLargePayload{
Supersedes: supersedes,
Bytes: payloadBytes,
MaxBytes: limit,
ResourceCount: len(snapshot.resourceOrder),
HydrateFrom: "/api/state",
},
})
if err != nil {
return payloadBytes, false, err
}
// Invalidate first. Even if the marker cannot queue, no later delta may be
// based on the payload the client did not receive.
c.stateSnapshot = nil
c.restRecovery = true
if !c.safeSend(marker) {
return payloadBytes, false, nil
}
return payloadBytes, true, nil
}
type preparedStateDelta struct {
data []byte
err error
}
func prepareStateDelta(previous, current *clientStateSnapshot) preparedStateDelta {
delta, err := buildClientStateDelta(previous, current)
if err != nil {
return preparedStateDelta{err: err}
}
if len(delta) == 0 {
return preparedStateDelta{}
}
data, err := json.Marshal(Message{Type: "rawData", Data: delta})
return preparedStateDelta{data: data, err: err}
}
// This map belongs to one serial dispatch, never the Hub or a tenant cache.
// Pointer identity identifies an actually accepted immutable snapshot, not a
// hash or resource ID. A reconnect/re-hydration gets its own baseline.
type stateDeltaDispatch map[*clientStateSnapshot]preparedStateDelta
func (c *Client) queueStateDelta(current *clientStateSnapshot) (int, bool, bool, error) {
return c.queueStateDeltaForDispatch(current, nil)
}
func (c *Client) queueStateDeltaForDispatch(current *clientStateSnapshot, dispatch stateDeltaDispatch) (int, bool, bool, error) {
if current == nil {
return 0, false, false, fmt.Errorf("current state snapshot is nil")
}
c.stateMu.Lock()
defer c.stateMu.Unlock()
if c.restRecovery {
dataLen, sent, markerErr := c.queueStateTooLargeLocked(
"rawData",
current,
0,
c.inboundLimitBytes(),
)
return dataLen, true, sent, markerErr
}
if c.stateSnapshot == nil {
// Initial hydration owns the baseline. A client still waiting for that
// payload must not receive a delta based on state it has never seen.
return 0, false, false, nil
}
prepared, found := dispatch[c.stateSnapshot]
if !found {
prepared = prepareStateDelta(c.stateSnapshot, current)
if dispatch != nil {
dispatch[c.stateSnapshot] = prepared
}
}
if prepared.err != nil {
return 0, true, false, prepared.err
}
if len(prepared.data) == 0 {
c.stateSnapshot = current
return 0, false, true, nil
}
data := prepared.data
if limit := c.inboundLimitBytes(); limit > 0 && int64(len(data)) > limit {
dataLen, sent, markerErr := c.queueStateTooLargeLocked("rawData", current, len(data), limit)
return dataLen, true, sent, markerErr
}
if !c.safeSend(data) {
return len(data), true, false, nil
}
c.stateSnapshot = current
return len(data), true, true, nil
}
// cloneAlertData returns a broadcast-safe copy of alert data to avoid data races when
// downstream sanitization/encoding happens concurrently with alert manager mutations.
func cloneAlertData(alert interface{}) interface{} {
switch a := alert.(type) {
case *alerts.Alert:
cloned := cloneAlert(a)
return cloned
case alerts.Alert:
cloned := cloneAlert(&a)
return cloned
default:
return alert
}
}
// cloneAlert performs a deep copy of the mutable fields within alerts.Alert.
func cloneAlert(src *alerts.Alert) alerts.Alert {
if src == nil {
return alerts.Alert{}
}
clone := *src
if src.AckTime != nil {
t := *src.AckTime
clone.AckTime = &t
}
if len(src.EscalationTimes) > 0 {
clone.EscalationTimes = append([]time.Time(nil), src.EscalationTimes...)
}
if src.Metadata != nil {
clone.Metadata = cloneMetadata(src.Metadata)
}
return clone
}
// cloneMetadata creates a deep copy of alert metadata to detach from shared maps/slices.
func cloneMetadata(src map[string]interface{}) map[string]interface{} {
if src == nil {
return nil
}
dst := make(map[string]interface{}, len(src))
for k, v := range src {
dst[k] = cloneMetadataValue(v)
}
return dst
}
func cloneMetadataValue(value interface{}) interface{} {
switch v := value.(type) {
case map[string]interface{}:
return cloneMetadata(v)
case map[string]string:
m := make(map[string]interface{}, len(v))
for key, val := range v {
m[key] = val
}
return m
case []interface{}:
arr := make([]interface{}, len(v))
for i, elem := range v {
arr[i] = cloneMetadataValue(elem)
}
return arr
case []string:
arr := make([]string, len(v))
copy(arr, v)
return arr
case []int:
arr := make([]int, len(v))
copy(arr, v)
return arr
case []float64:
arr := make([]float64, len(v))
copy(arr, v)
return arr
default:
return v
}
}
// TenantBroadcast represents a broadcast message targeted at a specific tenant.
type TenantBroadcast struct {
OrgID string
Message Message
}
type stateBroadcastRequest struct {
orgID string
}
// OrgAuthChecker checks if a user/token can access an organization.
type OrgAuthChecker interface {
// CanAccessOrg checks if the given user (and optional token) can access the org.
CanAccessOrg(userID string, token interface{}, orgID string) bool
}
// OrgAuthContextChecker is an OrgAuthChecker whose decision can depend on
// request-bound authority, such as a proxy-authenticated administrator.
type OrgAuthContextChecker interface {
CanAccessOrgContext(ctx context.Context, userID string, token interface{}, orgID string) bool
}
// MultiTenantCheckResult contains the result of a multi-tenant check.
type MultiTenantCheckResult struct {
Allowed bool
FeatureEnabled bool // false = feature flag disabled
Licensed bool // false = not licensed (only meaningful if FeatureEnabled is true)
Reason string // Human-readable reason for denial
}
// MultiTenantChecker checks if multi-tenant functionality is enabled and licensed.
type MultiTenantChecker interface {
// CheckMultiTenant checks if multi-tenant is enabled (feature flag) and licensed for the org.
// Returns a result with details about why access was denied if not allowed.
CheckMultiTenant(ctx context.Context, orgID string) MultiTenantCheckResult
}
// Hub maintains active WebSocket clients and broadcasts messages
type Hub struct {
clients map[*Client]bool // Flat client registry for lifecycle management
clientsByTenant map[string]map[*Client]bool // Per-tenant client tracking
broadcast chan []byte
broadcastSeq chan Message // Sequenced broadcast channel for ordering
tenantBroadcast chan TenantBroadcast // Per-tenant broadcast channel
register chan *Client
unregister chan *Client
stopChan chan struct{} // Signals shutdown
stopOnce sync.Once
runStarted chan struct{}
runDone chan struct{}
runStartOnce sync.Once
runDoneOnce sync.Once
initialStateWG sync.WaitGroup
stateBuildSlot chan struct{}
mu sync.RWMutex
getState func(orgID string) interface{} // Function to get state for specific tenant
allowedOrigins []string // Allowed origins for CORS
orgAuthChecker OrgAuthChecker // Org authorization checker
multiTenantChecker MultiTenantChecker // Multi-tenant feature flag and license checker
isTrustedProxy func(ip string) bool // Optional: checks if peer IP is a trusted reverse proxy
// Broadcast coalescing fields
coalesceWindow time.Duration
coalescePending *Message
coalesceReady *Message
coalesceTimer *time.Timer
coalesceGeneration uint64
nextGeneration uint64
coalesceMutex sync.Mutex
// Per-tenant coalescing
tenantCoalescePending map[string]*Message
tenantCoalesceReady map[string]*Message
tenantCoalesceTimers map[string]*time.Timer
tenantCoalesceGeneration map[string]uint64
stateBroadcastWake chan struct{}
stateBroadcastDone chan struct{}
// stateBroadcastInterval is the minimum spacing between state broadcasts
// to one audience (all clients, or one tenant); zero disables spacing.
stateBroadcastInterval time.Duration
lastStateBroadcast time.Time
tenantLastStateBroadcast map[string]time.Time
}
// defaultStateBroadcastInterval spaces state broadcasts to one audience. Each
// broadcast rebuilds and diffs the full frontend state, and every accepted
// agent report requests one, so a 100 ms debounce let broadcasts scale with
// agent count (#2199). Report-driven store refreshes run at most once per
// 2 seconds, so broadcasting more often mostly re-sent the same resources.
const defaultStateBroadcastInterval = 2 * time.Second
// Message represents a WebSocket message
type Message struct {
Type string `json:"type"`
Data interface{} `json:"data"`
Timestamp string `json:"timestamp,omitempty"`
}
// SetStateGetter sets the tenant-aware state getter function.
func (h *Hub) SetStateGetter(getState func(orgID string) interface{}) {
h.mu.Lock()
defer h.mu.Unlock()
h.getState = getState
}
// SetOrgAuthChecker sets the organization authorization checker.
func (h *Hub) SetOrgAuthChecker(checker OrgAuthChecker) {
h.mu.Lock()
defer h.mu.Unlock()
h.orgAuthChecker = checker
}
// SetMultiTenantChecker sets the multi-tenant feature flag and license checker.
func (h *Hub) SetMultiTenantChecker(checker MultiTenantChecker) {
h.mu.Lock()
defer h.mu.Unlock()
h.multiTenantChecker = checker
}
// SetTrustedProxyChecker sets the function used to verify whether a peer IP is a
// trusted reverse proxy. When set, X-Forwarded-Host/X-Forwarded-Proto are only
// trusted in checkOrigin when the peer passes this check.
func (h *Hub) SetTrustedProxyChecker(fn func(ip string) bool) {
h.mu.Lock()
defer h.mu.Unlock()
h.isTrustedProxy = fn
}
func (h *Hub) hasStateGetter() bool {
h.mu.RLock()
defer h.mu.RUnlock()
return h.getState != nil
}
// getStateForClient returns the state for a specific client based on their tenant
func (h *Hub) getStateForClient(client *Client) interface{} {
if client == nil {
return h.getStateForOrg("")
}
return h.getStateForOrg(client.orgID)
}
func (h *Hub) getStateForOrg(orgID string) interface{} {
h.mu.RLock()
getState := h.getState
h.mu.RUnlock()
if getState != nil {
return getState(normalizeOrgID(orgID))
}
return nil
}
// NewHub creates a new WebSocket hub
func NewHub(getState func(orgID string) interface{}) *Hub {
return &Hub{
clients: make(map[*Client]bool),
clientsByTenant: make(map[string]map[*Client]bool),
broadcast: make(chan []byte, 256),
broadcastSeq: make(chan Message, 256), // Buffered sequenced channel
tenantBroadcast: make(chan TenantBroadcast, 256), // Per-tenant broadcasts
register: make(chan *Client),
unregister: make(chan *Client),
stopChan: make(chan struct{}),
runStarted: make(chan struct{}),
runDone: make(chan struct{}),
stateBuildSlot: make(chan struct{}, 1),
getState: getState,
allowedOrigins: []string{}, // Default to empty (will be set based on actual host)
coalesceWindow: 100 * time.Millisecond, // Coalesce rapid updates within 100ms
tenantCoalescePending: make(map[string]*Message),
tenantCoalesceReady: make(map[string]*Message),
tenantCoalesceTimers: make(map[string]*time.Timer),
tenantCoalesceGeneration: make(map[string]uint64),
stateBroadcastWake: make(chan struct{}, 1),
stateBroadcastDone: make(chan struct{}),
stateBroadcastInterval: defaultStateBroadcastInterval,
tenantLastStateBroadcast: make(map[string]time.Time),
}
}
// Run starts the hub's main loop
func (h *Hub) Run() {
h.runStartOnce.Do(func() {
close(h.runStarted)
})
// Start broadcast sequencer goroutine
sequencerDone := make(chan struct{})
go func() {
defer close(sequencerDone)
h.runBroadcastSequencer()
}()
go h.runStateBroadcastWorker()
defer func() {
<-sequencerDone
<-h.stateBroadcastDone
h.runDoneOnce.Do(func() {
close(h.runDone)
})
}()
log.Info().Msg("WebSocket state payload configured for unified resources")
pingTicker := time.NewTicker(30 * time.Second)
defer pingTicker.Stop()
for {
select {
case client := <-h.register:
client.initializeLifecycle()
h.mu.Lock()
h.clients[client] = true
// Legacy clients belong to the default tenant.
orgID := normalizeOrgID(client.orgID)
if h.clientsByTenant[orgID] == nil {
h.clientsByTenant[orgID] = make(map[*Client]bool)
}
h.clientsByTenant[orgID][client] = true
h.mu.Unlock()
log.Info().Str("client", client.id).Str("org_id", client.orgID).Msg("webSocket client connected")
// Send initial state to the new client immediately
// Use tenant-aware state getter if available
hasGetState := h.hasStateGetter()
log.Debug().Bool("hasGetState", hasGetState).Msg("Checking getState function for new client")
if hasGetState {
h.initialStateWG.Add(1)
go func() {
defer h.initialStateWG.Done()
h.sendInitialState(client)
}()
} else {
log.Warn().
Str("component", websocketHubComponent).
Str("action", "initial_state_skipped_missing_getter").
Str("client", client.id).
Str("org_id", client.orgID).
Msg("No getState function defined")
}
case client := <-h.unregister:
h.mu.Lock()
if h.removeClientLocked(client) {
h.mu.Unlock()
log.Info().Str("client", client.id).Str("org_id", client.orgID).Msg("webSocket client disconnected")
} else {
h.mu.Unlock()
}
case message := <-h.broadcast:
h.dispatchToClients(message, "Client send channel full, dropping broadcast message and closing connection")
case <-pingTicker.C:
h.sendPing()
case <-h.stopChan:
log.Info().Msg("webSocket hub shutting down")
// Cancel delayed initial-state work first, then let every producer
// finish before closing client channels. Broadcast workers and
// initial-state delivery otherwise race a final safeSend against
// channel closure during shutdown.
h.mu.Lock()
for client := range h.clients {
client.closeLifecycle()
}
for _, tenantClients := range h.clientsByTenant {
for client := range tenantClients {
client.closeLifecycle()
}
}
h.mu.Unlock()
<-sequencerDone
<-h.stateBroadcastDone
h.initialStateWG.Wait()
h.mu.Lock()
for client := range h.clients {
client.closeSend()
}
for _, tenantClients := range h.clientsByTenant {
for client := range tenantClients {
client.closeSend()
}
}
h.clients = make(map[*Client]bool)
h.clientsByTenant = make(map[string]map[*Client]bool)
h.mu.Unlock()
return
}
}
}
// Stop gracefully shuts down the hub
func (h *Hub) Stop() {
h.stopOnce.Do(func() {
close(h.stopChan)
})
select {
case <-h.runStarted:
<-h.runDone
default:
}
h.initialStateWG.Wait()
}
func (h *Hub) isStopping() bool {
select {
case <-h.stopChan:
return true
default:
return false
}
}
func (h *Hub) waitForActiveClient(client *Client, delay time.Duration) bool {
timer := time.NewTimer(delay)
defer timer.Stop()
lifecycleDone := client.lifecycleSignal()
select {
case <-timer.C:
h.mu.RLock()
_, registered := h.clients[client]
h.mu.RUnlock()
return registered && !client.closed.Load() && !h.isStopping()
case <-lifecycleDone:
return false
case <-h.stopChan:
return false
}
}
func (h *Hub) sendInitialState(client *Client) {
log.Debug().Str("client", client.id).Msg("starting initial state goroutine")
if !h.waitForActiveClient(client, initialWelcomeDelay) {
log.Debug().Str("client", client.id).Msg("client disconnected before welcome message")
return
}
welcomeMsg := Message{
Type: "welcome",
Data: map[string]string{"message": "Connected to Pulse WebSocket", "orgId": client.orgID},
}
data, err := json.Marshal(welcomeMsg)
if err != nil {
log.Error().Err(err).Str("client", client.id).Msg("Failed to marshal welcome message")
return
}
log.Info().Str("client", client.id).Msg("sending welcome message")
if !client.safeSend(data) {
log.Warn().Str("client", client.id).Msg("failed to send welcome message - client closed or buffer full")
return
}
log.Info().Str("client", client.id).Msg("welcome message sent")
if !h.waitForActiveClient(client, initialStateMessageDelay) {
log.Debug().Str("client", client.id).Msg("client disconnected before initial state")
return
}
if !h.acquireStateBuildSlot(client) {
log.Debug().Str("client", client.id).Msg("client disconnected while waiting to build initial state")
return
}
defer h.releaseStateBuildSlot()
log.Debug().Str("client", client.id).Msg("about to get state")
stateData := h.getStateForClient(client)
log.Debug().Str("client", client.id).Interface("stateType", fmt.Sprintf("%T", stateData)).Msg("got state for initial message")
stateData = h.prepareStateForBroadcast(stateData)
h.mu.RLock()
_, stillRegistered := h.clients[client]
h.mu.RUnlock()
if !stillRegistered {
log.Debug().Str("client", client.id).Msg("client disconnected before initial state")
return
}
dataLen, sent, err := client.queueFullState("initialState", stateData)
if err != nil {
log.Error().Err(err).Str("client", client.id).Msg("failed to marshal initial state")
return
}
log.Info().Str("client", client.id).Int("dataLen", dataLen).Int("dataKB", dataLen/1024).Msg("sending initial state to client")
if sent {
log.Info().Str("client", client.id).Msg("initial state sent successfully")
} else {
log.Warn().Str("client", client.id).Msg("client closed or buffer full, skipping initial state")
}
}
func (h *Hub) sendRequestedState(client *Client) {
if client == nil || !h.hasStateGetter() {
return
}
if !h.acquireStateBuildSlot(client) {
log.Debug().Str("client", client.id).Msg("client disconnected while waiting to build requested state")
return
}
defer h.releaseStateBuildSlot()
_, sent, err := client.queueFullState(
"rawData",
h.prepareStateForBroadcast(h.getStateForClient(client)),
)
if err != nil {
log.Error().Err(err).Str("client", client.id).Msg("failed to marshal state for requestData")
return
}
if !sent {
log.Warn().Str("client", client.id).Msg("Failed to queue requestData state response; client channel closed or full")
}
}
func (h *Hub) acquireStateBuildSlot(client *Client) bool {
if client == nil {
select {
case h.stateBuildSlot <- struct{}{}:
return true
case <-h.stopChan:
return false
}
}
lifecycleDone := client.lifecycleSignal()
select {
case h.stateBuildSlot <- struct{}{}:
h.mu.RLock()
_, registered := h.clients[client]
h.mu.RUnlock()
if registered && !client.closed.Load() && !h.isStopping() {
return true
}
h.releaseStateBuildSlot()
return false
case <-lifecycleDone:
return false
case <-h.stopChan:
return false
}
}
func (h *Hub) releaseStateBuildSlot() {
<-h.stateBuildSlot
}
func (h *Hub) tryRegisterClient(client *Client) bool {
if h.isStopping() {
return false
}
select {
case h.register <- client:
return true
case <-h.stopChan:
return false
}
}
// HandleWebSocket handles WebSocket upgrade requests
func (h *Hub) HandleWebSocket(w http.ResponseWriter, r *http.Request) {
h.HandleWebSocketWithAuthorization(w, r, nil)
}
// HandleWebSocketWithAuthorization keeps an admitted connection subject to
// current authority. The callback runs only on its writer, before every frame
// (including queued frames and idle pings). Nil preserves existing callers.
func (h *Hub) HandleWebSocketWithAuthorization(w http.ResponseWriter, r *http.Request, authorize func() bool) {
log.Info().
Str("origin", r.Header.Get("Origin")).
Str("host", r.Host).
Str("userAgent", r.Header.Get("User-Agent")).
Msg("WebSocket upgrade request")
// Extract org ID from request for tenant isolation
// Priority: Header > Cookie > Query param > Default
orgID := strings.TrimSpace(r.Header.Get("X-Pulse-Org-ID"))
if orgID == "" {
if cookie, err := r.Cookie("pulse_org_id"); err == nil {
orgID = strings.TrimSpace(cookie.Value)
}
}
if orgID == "" {
orgID = strings.TrimSpace(r.URL.Query().Get("org_id"))
}
if !isValidWebSocketOrgID(orgID) {
log.Warn().
Int("org_id_len", len(orgID)).
Msg("WebSocket connection denied - invalid organization ID")
http.Error(w, "Invalid organization ID", http.StatusBadRequest)
return
}
// Multi-tenant feature flag and license check for non-default orgs
h.mu.RLock()
mtChecker := h.multiTenantChecker
authChecker := h.orgAuthChecker
h.mu.RUnlock()
if orgID != "default" {
// Check if multi-tenant is enabled and licensed
if mtChecker != nil {
result := mtChecker.CheckMultiTenant(r.Context(), orgID)
if !result.Allowed {
userID := getUserFromContext(r.Context())
audit.Log(
"websocket_multitenant_access_denied",
userID,
extractPeerIP(r.RemoteAddr),
r.URL.Path,
false,
fmt.Sprintf("org_id=%s reason=%s feature_enabled=%t licensed=%t", orgID, result.Reason, result.FeatureEnabled, result.Licensed),
)
log.Warn().
Str("org_id", orgID).
Bool("feature_enabled", result.FeatureEnabled).
Bool("licensed", result.Licensed).
Str("reason", result.Reason).
Msg("WebSocket connection denied - multi-tenant check failed")
if !result.FeatureEnabled {
// Feature flag disabled - 501 Not Implemented
http.Error(w, "Multi-tenant functionality is not enabled", http.StatusNotImplemented)
} else {
// Feature enabled but not licensed - 402 Payment Required
http.Error(w, "Multi-tenant access requires an Enterprise license", http.StatusPaymentRequired)
}
return
}
}
}
// Authorization check - auth context should already be set by
// AuthContextMiddleware. Non-default organizations always require it. The
// default organization is checked whenever a principal is present: in
// multi-organization mode it is not open to every signed-in user, while an
// installation without authentication has no principal to check.
userID := getUserFromContext(r.Context())
token := getAPITokenFromContext(r.Context())
if authChecker != nil && (orgID != "default" || userID != "" || token != nil) {
allowed := false
if contextChecker, ok := authChecker.(OrgAuthContextChecker); ok {
allowed = contextChecker.CanAccessOrgContext(r.Context(), userID, token, orgID)
} else {
allowed = authChecker.CanAccessOrg(userID, token, orgID)
}
if !allowed {
audit.Log(
"websocket_org_access_denied",
userID,
extractPeerIP(r.RemoteAddr),
r.URL.Path,
false,
fmt.Sprintf("org_id=%s", orgID),
)
log.Warn().
Str("org_id", orgID).
Str("user_id", userID).
Msg("WebSocket connection denied - unauthorized org access")
http.Error(w, "Unauthorized to access organization", http.StatusForbidden)
return
}
}
// Create upgrader with our origin check.
// Buffers are 64 KB: gorilla streams larger payloads across the buffer
// automatically, so the previous 4 MB allocation per connection was
// unnecessary overhead that scaled badly with concurrent clients
// (100 clients × 8 MB = 800 MB just in buffers).
upgrader := websocket.Upgrader{
ReadBufferSize: 64 * 1024,
WriteBufferSize: 64 * 1024,
CheckOrigin: h.checkOrigin,
EnableCompression: true,
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Error().Err(err).Msg("failed to upgrade WebSocket connection")
return
}
conn.EnableWriteCompression(true)
if err := conn.SetCompressionLevel(flate.BestSpeed); err != nil {
log.Warn().Err(err).Msg("Failed to set WebSocket compression level")
}
clientID := utils.GenerateID("client")
client := &Client{
hub: h,
conn: conn,
orgID: orgID,
// Keep buffer bounded to avoid holding large state snapshots indefinitely if a client stalls.
send: make(chan []byte, 128),
id: clientID,
lastPing: time.Now(),
authorizeWrite: authorize,
}
// This must be captured from the upgrade URL before registration: initial
// state construction begins as soon as the hub accepts the client.
client.applyAdvertisedInboundLimit(r.URL.Query().Get(inboundLimitQueryParam))
log.Info().Str("client", clientID).Str("org_id", orgID).Msg("webSocket client created")
if !client.hub.tryRegisterClient(client) {
log.Info().Str("client", clientID).Str("org_id", orgID).Msg("WebSocket hub stopping; rejecting new client")
conn.Close()
return
}
// Start goroutines for reading and writing
go client.writePump()
go client.readPump()
}
// getUserFromContext extracts the user ID from request context.
func getUserFromContext(ctx context.Context) string {
return auth.GetUser(ctx)
}
// getAPITokenFromContext extracts the API token from request context.
func getAPITokenFromContext(ctx context.Context) interface{} {
return auth.GetAPIToken(ctx)
}
func normalizeOrgID(orgID string) string {
orgID = strings.TrimSpace(orgID)
if orgID == "" {
return "default"
}
return orgID
}
// removeClientLocked removes a client from every hub index and closes the send
// channel once. h.mu must be held by the caller.
func (h *Hub) removeClientLocked(client *Client) bool {
if client == nil {
return false
}
removed := false
if _, ok := h.clients[client]; ok {
delete(h.clients, client)
removed = true
}
if client.orgID != "" {
if tenantClients := h.clientsByTenant[client.orgID]; tenantClients != nil {
if _, ok := tenantClients[client]; ok {
delete(tenantClients, client)
removed = true
if len(tenantClients) == 0 {
delete(h.clientsByTenant, client.orgID)
}
}
}
} else {
for orgID, tenantClients := range h.clientsByTenant {
if _, ok := tenantClients[client]; ok {
delete(tenantClients, client)
removed = true
if len(tenantClients) == 0 {
delete(h.clientsByTenant, orgID)
}
}
}
}
if removed {
client.closeLifecycle()
client.closeSend()
}
return removed
}
// dispatchToClients fan-outs a marshaled payload to all clients, dropping any that
// cannot keep up to prevent unbounded buffering.
func (h *Hub) dispatchToClients(data []byte, dropLog string) {
h.mu.RLock()
clients := make([]*Client, 0, len(h.clients))
for client := range h.clients {
clients = append(clients, client)
}
h.mu.RUnlock()
for _, client := range clients {
if !client.safeSend(data) {
h.mu.Lock()
if h.removeClientLocked(client) {
log.Warn().Str("client", client.id).Msg(dropLog)
}
h.mu.Unlock()
}
}
}
func (h *Hub) markGlobalStateBroadcastReady(generation uint64) {
h.coalesceMutex.Lock()
if generation != h.coalesceGeneration || h.coalescePending == nil {
h.coalesceMutex.Unlock()
return
}
h.coalesceReady = h.coalescePending
h.coalescePending = nil
h.coalesceTimer = nil
h.lastStateBroadcast = time.Now()
h.coalesceMutex.Unlock()
h.wakeStateBroadcastWorker()
}
func (h *Hub) markTenantStateBroadcastReady(orgID string, generation uint64) {
h.coalesceMutex.Lock()
if generation != h.tenantCoalesceGeneration[orgID] {
h.coalesceMutex.Unlock()
return
}
pending := h.tenantCoalescePending[orgID]
if pending == nil {
h.coalesceMutex.Unlock()
return
}
h.tenantCoalesceReady[orgID] = pending
delete(h.tenantCoalescePending, orgID)
delete(h.tenantCoalesceTimers, orgID)
delete(h.tenantCoalesceGeneration, orgID)
h.tenantLastStateBroadcast[orgID] = time.Now()
h.coalesceMutex.Unlock()
h.wakeStateBroadcastWorker()
}
func (h *Hub) wakeStateBroadcastWorker() {
select {
case h.stateBroadcastWake <- struct{}{}:
default:
}
}
func (h *Hub) popReadyStateBroadcasts() (*Message, map[string]*Message) {
h.coalesceMutex.Lock()
defer h.coalesceMutex.Unlock()
global := h.coalesceReady
h.coalesceReady = nil
tenants := h.tenantCoalesceReady
h.tenantCoalesceReady = make(map[string]*Message)
return global, tenants
}
func (h *Hub) dispatchStateBroadcast(pending *Message, orgID string) {
if pending == nil || h.isStopping() || !h.hasRecipientsForMessage(*pending, orgID) {
return
}
if !h.acquireStateBuildSlot(nil) {
return
}
defer h.releaseStateBuildSlot()
if _, currentStateRequest := h.messageHasCurrentStateRequest(*pending); currentStateRequest {
prepared, ok := h.prepareMessageForMarshal(*pending, orgID)
if !ok {
return
}
h.dispatchCurrentStateSnapshot(prepared.Data, orgID)
return
}
data, ok := h.marshalBroadcastMessage(*pending, orgID)
if !ok {
return
}
if orgID == "" {
h.dispatchToClients(data, "Client send channel full, dropping coalesced message and closing connection")
return
}
h.dispatchToTenantClients(orgID, data, "Client send channel full, dropping tenant coalesced message and closing connection")
}
func (h *Hub) dispatchCurrentStateSnapshot(state interface{}, orgID string) {
snapshot, err := buildClientStateSnapshot(state)
if err != nil {
log.Error().Err(err).Str("org_id", normalizeOrgID(orgID)).Msg("Failed to build WebSocket state delta snapshot")
return
}
h.mu.RLock()
clients := make([]*Client, 0, len(h.clients))
if orgID == "" {
for client := range h.clients {
clients = append(clients, client)
}
} else if tenantClients := h.clientsByTenant[orgID]; tenantClients != nil {
for client := range tenantClients {
clients = append(clients, client)
}
}
h.mu.RUnlock()
dispatch := make(stateDeltaDispatch)
for _, client := range clients {
dataLen, attempted, sent, queueErr := client.queueStateDeltaForDispatch(snapshot, dispatch)
if queueErr != nil {
log.Error().Err(queueErr).Str("client", client.id).Str("org_id", client.orgID).Msg("Failed to marshal WebSocket state delta")
continue
}
if !attempted || sent {
if sent {
log.Debug().Str("client", client.id).Int("dataLen", dataLen).Msg("queued WebSocket state delta")
}
continue
}
h.mu.Lock()
if h.removeClientLocked(client) {
log.Warn().Str("client", client.id).Str("org_id", client.orgID).Msg("Client send channel full, dropping state delta and closing connection")
}
h.mu.Unlock()
}
}
func (h *Hub) runStateBroadcastWorker() {
defer close(h.stateBroadcastDone)
for {
select {
case <-h.stateBroadcastWake:
global, tenants := h.popReadyStateBroadcasts()
h.dispatchStateBroadcast(global, "")
for orgID, pending := range tenants {
h.dispatchStateBroadcast(pending, orgID)
}
case <-h.stopChan:
return
}
}
}
func (h *Hub) messageHasCurrentStateRequest(msg Message) (stateBroadcastRequest, bool) {
req, ok := msg.Data.(stateBroadcastRequest)
return req, ok
}
func (h *Hub) hasRecipientsForMessage(msg Message, orgID string) bool {
if _, ok := h.messageHasCurrentStateRequest(msg); !ok {
return true
}
if orgID != "" {
return h.GetTenantClientCount(orgID) > 0
}
return h.GetClientCount() > 0
}
func (h *Hub) prepareMessageForMarshal(msg Message, orgID string) (Message, bool) {
req, ok := h.messageHasCurrentStateRequest(msg)
if !ok {
return msg, true
}
requestOrgID := strings.TrimSpace(req.orgID)
if requestOrgID == "" {
requestOrgID = orgID
}
h.mu.RLock()
hasGetter := h.getState != nil
h.mu.RUnlock()
if !hasGetter {
log.Warn().
Str("component", websocketHubComponent).
Str("action", "current_state_broadcast_skipped_missing_getter").
Str("org_id", normalizeOrgID(requestOrgID)).
Msg("Skipping current state broadcast because no state getter is configured")
return Message{}, false
}
msg.Data = h.prepareStateForBroadcast(h.getStateForOrg(requestOrgID))
return msg, true
}
func (h *Hub) marshalBroadcastMessage(msg Message, orgID string) ([]byte, bool) {
prepared, ok := h.prepareMessageForMarshal(msg, orgID)
if !ok {
return nil, false
}
data, err := json.Marshal(prepared)
if err != nil {
if orgID != "" {
log.Error().Err(err).Str("org_id", orgID).Str("type", prepared.Type).Msg("Failed to marshal tenant broadcast message")
} else {
log.Error().Err(err).Str("type", prepared.Type).Msg("Failed to marshal broadcast message")
}
return nil, false
}
return data, true
}
// stateBroadcastDelayLocked returns how long a newly requested state
// broadcast waits: coalesceWindow to merge simultaneous signals, and at least
// stateBroadcastInterval after the audience's previous broadcast. The caller
// holds coalesceMutex.
func (h *Hub) stateBroadcastDelayLocked(last time.Time) time.Duration {
delay := h.coalesceWindow
if !last.IsZero() && h.stateBroadcastInterval > 0 {
if wait := h.stateBroadcastInterval - time.Since(last); wait > delay {
delay = wait
}
}
return delay
}
// runBroadcastSequencer handles sequenced broadcasts with coalescing for rapid state updates
func (h *Hub) runBroadcastSequencer() {
for {
select {
case msg := <-h.broadcastSeq:
// Handle raw data (state) messages with coalescing
if msg.Type == "rawData" {
h.coalesceMutex.Lock()
// The latest state wins. A due broadcast is not rescheduled, so
// a steady stream of signals cannot postpone it.
current := msg
h.coalescePending = ¤t
if h.coalesceTimer == nil {
h.nextGeneration++
generation := h.nextGeneration
h.coalesceGeneration = generation
h.coalesceTimer = time.AfterFunc(h.stateBroadcastDelayLocked(h.lastStateBroadcast), func() {
h.markGlobalStateBroadcastReady(generation)
})
}
h.coalesceMutex.Unlock()
} else {
// Non-state messages (alerts, etc.) - send immediately
if data, ok := h.marshalBroadcastMessage(msg, ""); ok {
h.dispatchToClients(data, "Client send channel full, dropping message and closing connection")
}
}
case tb := <-h.tenantBroadcast:
// Handle tenant-specific broadcasts with coalescing
if tb.Message.Type == "rawData" {
h.coalesceMutex.Lock()
// The latest state wins; a due tenant broadcast keeps its time.
msgCopy := tb.Message
h.tenantCoalescePending[tb.OrgID] = &msgCopy
if h.tenantCoalesceTimers[tb.OrgID] == nil {
h.nextGeneration++
generation := h.nextGeneration
h.tenantCoalesceGeneration[tb.OrgID] = generation
orgID := tb.OrgID // Capture for closure
h.tenantCoalesceTimers[orgID] = time.AfterFunc(h.stateBroadcastDelayLocked(h.tenantLastStateBroadcast[orgID]), func() {
h.markTenantStateBroadcastReady(orgID, generation)
})
}
h.coalesceMutex.Unlock()
} else {
// Non-state messages - send immediately to tenant
if data, ok := h.marshalBroadcastMessage(tb.Message, tb.OrgID); ok {
h.dispatchToTenantClients(tb.OrgID, data, "Client send channel full, dropping tenant message and closing connection")
}
}
case <-h.stopChan:
log.Debug().Msg("broadcast sequencer shutting down")
// Cancel pending timer if exists
h.coalesceMutex.Lock()
if h.coalesceTimer != nil {
h.coalesceTimer.Stop()
}
// Cancel all tenant timers
for _, timer := range h.tenantCoalesceTimers {
if timer != nil {
timer.Stop()
}
}
h.coalesceMutex.Unlock()
return
}
}
}
// BroadcastState broadcasts state update to all clients via sequencer
func (h *Hub) BroadcastState(state interface{}) {
if h.isStopping() {
log.Debug().Msg("Skipping state broadcast while hub is stopping")
return
}
log.Debug().Msg("broadcasting state")
stateData := h.prepareStateForBroadcast(state)
msg := Message{
Type: "rawData",
Data: stateData,
}
// Send through sequencer for ordering and coalescing
select {
case h.broadcastSeq <- msg:
default:
log.Warn().
Str("component", websocketHubComponent).
Str("action", "enqueue_state_dropped").
Str("channel", "broadcast_seq").
Int("channel_depth", len(h.broadcastSeq)).
Int("channel_capacity", cap(h.broadcastSeq)).
Msg("Broadcast sequencer channel full, dropping state update")
}
}
// BroadcastCurrentState coalesces a state-change signal and resolves the current
// tenant-aware state only when the broadcast window is ready to flush.
func (h *Hub) BroadcastCurrentState() {
if h.isStopping() {
log.Debug().Msg("Skipping current state broadcast while hub is stopping")
return
}
log.Debug().Msg("broadcasting current state")
msg := Message{
Type: "rawData",
Data: stateBroadcastRequest{},
}
select {
case h.broadcastSeq <- msg:
default:
log.Warn().
Str("component", websocketHubComponent).
Str("action", "enqueue_state_dropped").
Str("channel", "broadcast_seq").
Int("channel_depth", len(h.broadcastSeq)).
Int("channel_capacity", cap(h.broadcastSeq)).
Msg("Broadcast sequencer channel full, dropping state update")
}
}
// BroadcastStateToTenant broadcasts state update only to clients of a specific tenant.
func (h *Hub) BroadcastStateToTenant(orgID string, state interface{}) {
if h.isStopping() {
log.Debug().Str("org_id", orgID).Msg("Skipping tenant state broadcast while hub is stopping")
return
}
log.Debug().Str("org_id", orgID).Msg("Broadcasting state to tenant")
stateData := h.prepareStateForBroadcast(state)
msg := Message{
Type: "rawData",
Data: stateData,
}
// Send through tenant broadcast channel
select {
case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
default:
log.Warn().
Str("component", websocketHubComponent).
Str("action", "enqueue_tenant_state_dropped").
Str("org_id", orgID).
Str("channel", "tenant_broadcast").
Int("channel_depth", len(h.tenantBroadcast)).
Int("channel_capacity", cap(h.tenantBroadcast)).
Msg("Tenant broadcast channel full, dropping state update")
}
}
// BroadcastCurrentStateToTenant coalesces a tenant state-change signal and
// resolves that tenant's current state only when the broadcast window flushes.
func (h *Hub) BroadcastCurrentStateToTenant(orgID string) {
orgID = normalizeOrgID(orgID)
if h.isStopping() {
log.Debug().Str("org_id", orgID).Msg("Skipping tenant current state broadcast while hub is stopping")
return
}
log.Debug().Str("org_id", orgID).Msg("Broadcasting current state to tenant")
msg := Message{
Type: "rawData",
Data: stateBroadcastRequest{orgID: orgID},
}
select {
case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
default:
log.Warn().
Str("component", websocketHubComponent).
Str("action", "enqueue_tenant_state_dropped").
Str("org_id", orgID).
Str("channel", "tenant_broadcast").
Int("channel_depth", len(h.tenantBroadcast)).
Int("channel_capacity", cap(h.tenantBroadcast)).
Msg("Tenant broadcast channel full, dropping state update")
}
}
// dispatchToTenantClients sends a message only to clients of a specific tenant.
func (h *Hub) dispatchToTenantClients(orgID string, data []byte, dropLog string) {
h.mu.RLock()
tenantClients := h.clientsByTenant[orgID]
if tenantClients == nil {
h.mu.RUnlock()
return
}
clients := make([]*Client, 0, len(tenantClients))
for client := range tenantClients {
clients = append(clients, client)
}
h.mu.RUnlock()
for _, client := range clients {
if !client.safeSend(data) {
h.mu.Lock()
if h.removeClientLocked(client) {
log.Warn().Str("client", client.id).Str("org_id", client.orgID).Msg(dropLog)
}
h.mu.Unlock()
}
}
}
// prepareStateForBroadcast applies websocket payload compatibility rules to state payloads.
func (h *Hub) prepareStateForBroadcast(state interface{}) interface{} {
return state
}
// GetTenantClientCount returns the number of connected clients for a specific tenant.
func (h *Hub) GetTenantClientCount(orgID string) int {
orgID = normalizeOrgID(orgID)
h.mu.RLock()
defer h.mu.RUnlock()
if tenantClients := h.clientsByTenant[orgID]; tenantClients != nil {
return len(tenantClients)
}
return 0
}
// BroadcastAlertToTenant broadcasts alert to clients of a specific tenant only.
// Empty org IDs are normalized to the default tenant.
func (h *Hub) BroadcastAlertToTenant(orgID string, alert interface{}) {
orgID = normalizeOrgID(orgID)
log.Info().Str("org_id", orgID).Msg("broadcasting alert to tenant WebSocket clients")
msg := Message{
Type: "alert",
Data: cloneAlertData(alert),
}
if h.isStopping() {
log.Debug().Str("org_id", orgID).Msg("Skipping tenant alert broadcast while hub is stopping")
return
}
select {
case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
default:
log.Warn().
Str("component", websocketHubComponent).
Str("action", "enqueue_tenant_alert_dropped").
Str("org_id", orgID).
Str("channel", "tenant_broadcast").
Int("channel_depth", len(h.tenantBroadcast)).
Int("channel_capacity", cap(h.tenantBroadcast)).
Msg("Tenant broadcast channel full, dropping alert")
}
}
// BroadcastAlertResolvedToTenant broadcasts alert resolution to clients of a specific tenant only.
// Empty org IDs are normalized to the default tenant.
func (h *Hub) BroadcastAlertResolvedToTenant(orgID string, alertID string) {
orgID = normalizeOrgID(orgID)
log.Info().Str("org_id", orgID).Str("alertID", alertID).Msg("broadcasting alert resolved to tenant WebSocket clients")
msg := Message{
Type: "alertResolved",
Data: map[string]string{
"alertIdentifier": alertID,
},
}
if h.isStopping() {
log.Debug().Str("org_id", orgID).Msg("Skipping tenant alert resolved broadcast while hub is stopping")
return
}
select {
case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
default:
log.Warn().
Str("component", websocketHubComponent).
Str("action", "enqueue_tenant_alert_resolved_dropped").
Str("org_id", orgID).
Str("channel", "tenant_broadcast").
Int("channel_depth", len(h.tenantBroadcast)).
Int("channel_capacity", cap(h.tenantBroadcast)).
Msg("Tenant broadcast channel full, dropping alert resolved")
}
}
// GetClientCount returns the number of connected clients
func (h *Hub) GetClientCount() int {
h.mu.RLock()
defer h.mu.RUnlock()
return len(h.clients)
}
// BroadcastToTenant sends a custom message to its owning organization.
func (h *Hub) BroadcastToTenant(orgID string, data interface{}) {
h.BroadcastMessageToTenant(orgID, Message{
Type: "custom",
Data: data,
Timestamp: time.Now().Format(time.RFC3339),
})
}
// BroadcastMessageToTenant delivers an event only to its owning organization.
// Empty organization IDs identify the default tenant, including single-tenant installs.
func (h *Hub) BroadcastMessageToTenant(orgID string, msg Message) {
orgID = normalizeOrgID(orgID)
if h.isStopping() {
return
}
msg.Data = sanitizeData(msg.Data)
select {
case h.tenantBroadcast <- TenantBroadcast{OrgID: orgID, Message: msg}:
default:
log.Warn().Str("component", websocketHubComponent).
Str("action", "enqueue_tenant_message_dropped").Str("org_id", orgID).
Str("message_type", msg.Type).Str("channel", "tenant_broadcast").
Int("channel_depth", len(h.tenantBroadcast)).
Int("channel_capacity", cap(h.tenantBroadcast)).
Msg("Tenant broadcast channel full, dropping event")
}
}
// BroadcastMessage accepts only instance-wide events. Tenant-derived messages
// must use BroadcastMessageToTenant with the organization owning their data.
func (h *Hub) BroadcastMessage(msg Message) {
switch msg.Type {
case "ping", "settingsUpdate", "update:progress":
// Heartbeats, shared system theme, and server software update progress
// describe the instance rather than any organization's infrastructure.
default:
log.Error().Str("type", msg.Type).Msg("Refusing unscoped WebSocket event")
return
}
if h.isStopping() {
log.Debug().Str("type", msg.Type).Msg("Skipping websocket broadcast while hub is stopping")
return
}
// Sanitize the message data to handle NaN values
msg.Data = sanitizeData(msg.Data)
data, err := json.Marshal(msg)
if err != nil {
log.Error().Err(err).Str("type", msg.Type).Msg("failed to marshal WebSocket message")
// Try to marshal without data to see what's failing
debugMsg := Message{Type: msg.Type, Data: "[error marshaling data]"}
if debugData, debugErr := json.Marshal(debugMsg); debugErr == nil {
log.Debug().Str("debugMsg", string(debugData)).Msg("debug message")
}
return
}
select {
case h.broadcast <- data:
default:
log.Warn().
Str("component", websocketHubComponent).
Str("action", "enqueue_broadcast_dropped").
Str("message_type", msg.Type).
Str("channel", "broadcast").
Int("channel_depth", len(h.broadcast)).
Int("channel_capacity", cap(h.broadcast)).
Msg("WebSocket broadcast channel full")
}
}
// sendPing sends a ping message to all clients
func (h *Hub) sendPing() {
msg := Message{
Type: "ping",
Data: map[string]int64{"timestamp": time.Now().Unix()},
}
h.BroadcastMessage(msg)
}
func isUnexpectedClientClose(err error) bool {
return websocket.IsUnexpectedCloseError(
err,
websocket.CloseNormalClosure,
websocket.CloseGoingAway,
websocket.CloseAbnormalClosure,
)
}
// Gorilla's read limit counts encoded frame bytes. Bound the decompressed
// reader as well, before buffering or JSON decoding, including messages whose
// sender has not finished the final frame.
func (c *Client) readInboundMessage() ([]byte, error) {
_, reader, err := c.conn.NextReader()
if err != nil {
return nil, err
}
message, err := io.ReadAll(io.LimitReader(reader, maxWebSocketInboundMessageSize+1))
if err != nil {
return nil, err
}
if len(message) > maxWebSocketInboundMessageSize {
if err := c.conn.WriteControl(websocket.CloseMessage,
websocket.FormatCloseMessage(websocket.CloseMessageTooBig, ""),
time.Now().Add(time.Second)); err != nil {
log.Debug().Err(err).Str("client", c.id).Msg("Failed to send WebSocket size-limit close")
}
return nil, websocket.ErrReadLimit
}
return message, nil
}
// readPump handles incoming messages from the client
func (c *Client) readPump() {
defer func() {
log.Info().Str("client", c.id).Msg("readPump exiting")
c.hub.unregister <- c
if err := c.conn.Close(); err != nil {
log.Debug().Err(err).Str("client", c.id).Msg("Failed to close WebSocket connection in readPump")
}
}()
if err := c.conn.SetReadDeadline(time.Now().Add(clientReadDeadline)); err != nil {
log.Warn().Err(err).Str("client", c.id).Msg("failed to set initial read deadline")
}
c.conn.SetReadLimit(maxWebSocketInboundMessageSize)
c.conn.SetPongHandler(func(string) error {
if err := c.conn.SetReadDeadline(time.Now().Add(clientReadDeadline)); err != nil {
log.Warn().Err(err).Str("client", c.id).Msg("failed to refresh read deadline on pong")
}
c.lastPing = time.Now()
return nil
})
log.Info().Str("client", c.id).Msg("readPump started")
for {
message, err := c.readInboundMessage()
if err != nil {
if isUnexpectedClientClose(err) {
log.Error().Err(err).Str("client", c.id).Msg("webSocket read error")
} else {
log.Info().Err(err).Str("client", c.id).Msg("webSocket closed")
}
break
}
// Any successful read proves the peer is alive; without this refresh a
// client whose pong frames are lost in transit dies at the deadline even
// while its 30s JSON heartbeats are arriving.
if err := c.conn.SetReadDeadline(time.Now().Add(clientReadDeadline)); err != nil {
log.Warn().Err(err).Str("client", c.id).Msg("failed to refresh read deadline on message")
}
// Handle incoming messages
var msg Message
if err := json.Unmarshal(message, &msg); err != nil {
log.Error().Err(err).Str("client", c.id).Msg("failed to unmarshal WebSocket message")
continue
}
// Handle different message types
switch msg.Type {
case "ping":
// Respond with pong
pong := Message{
Type: "pong",
Data: map[string]int64{"timestamp": time.Now().Unix()},
}
if data, err := json.Marshal(pong); err == nil {
if !c.safeSend(data) {
log.Warn().Str("client", c.id).Msg("Failed to queue pong response; client channel closed or full")
}
} else {
log.Error().Err(err).Str("client", c.id).Msg("Failed to marshal pong response")
}
case "requestData":
c.hub.sendRequestedState(c)
default:
log.Debug().Str("client", c.id).Str("type", msg.Type).Msg("received WebSocket message")
}
}
}
// allowWrite checks current authority without disclosing authentication detail.
func (c *Client) allowWrite() bool {
if c.authorizeWrite == nil || c.authorizeWrite() {
return true
}
// Never send an HTTP error or queued data after authority ends. The fixed
// policy close is understood by the existing frontend authentication path.
_ = c.conn.WriteControl(websocket.CloseMessage,
websocket.FormatCloseMessage(websocket.ClosePolicyViolation, "Monitoring access ended"),
time.Now().Add(10*time.Second))
return false
}
// writePump handles outgoing messages to the client.
func (c *Client) writePump() {
// Maximum consecutive write failures before disconnecting.
// This provides graceful degradation for slow clients.
const maxWriteFailures = 3
// Write deadline for messages. Increased from 10s to 30s to handle
// large state payloads on slower connections (e.g., Raspberry Pi, slow networks).
const writeDeadline = 30 * time.Second
// Ping deadline can be shorter since pings are small
const pingDeadline = 10 * time.Second
ticker := time.NewTicker(clientPingPeriod)
defer func() {
log.Info().Str("client", c.id).Msg("writePump exiting")
ticker.Stop()
if err := c.conn.Close(); err != nil {
log.Debug().Err(err).Str("client", c.id).Msg("Failed to close WebSocket connection in writePump")
}
}()
log.Info().Str("client", c.id).Msg("writePump started")
for {
select {
case message, ok := <-c.send:
if err := c.conn.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil {
log.Warn().Err(err).Str("client", c.id).Msg("failed to set write deadline before message send")
}
if !ok {
log.Debug().Str("client", c.id).Msg("send channel closed")
if err := c.conn.WriteMessage(websocket.CloseMessage, []byte{}); err != nil {
log.Warn().Err(err).Str("client", c.id).Msg("failed to send close message")
}
return
}
if !c.allowWrite() {
return
}
// Send the primary message
if err := c.conn.WriteMessage(websocket.TextMessage, message); err != nil {
c.writeFailures++
log.Warn().
Err(err).
Str("client", c.id).
Int("msgSize", len(message)).
Int32("consecutiveFailures", c.writeFailures).
Msg("Failed to write message")
// Graceful degradation: only disconnect after multiple consecutive failures
if c.writeFailures >= maxWriteFailures {
log.Error().
Str("client", c.id).
Int32("failures", c.writeFailures).
Msg("Too many consecutive write failures, disconnecting client")
return
}
// Skip this message and continue - don't disconnect immediately
continue
}
// Reset failure count on successful write
c.writeFailures = 0
// Send any queued messages
n := len(c.send)
flushLoop:
for i := 0; i < n; i++ {
select {
case msg, ok := <-c.send:
if !ok || !c.allowWrite() {
return
}
if err := c.conn.WriteMessage(websocket.TextMessage, msg); err != nil {
log.Warn().Err(err).Str("client", c.id).Int("msgSize", len(msg)).Msg("failed to flush queued message")
// Don't disconnect on queued message failure, just break the flush loop
break flushLoop
}
default:
// No more messages
}
}
case <-ticker.C:
if !c.allowWrite() {
return
}
if err := c.conn.SetWriteDeadline(time.Now().Add(pingDeadline)); err != nil {
log.Warn().Err(err).Str("client", c.id).Msg("failed to set write deadline for ping")
}
if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
log.Debug().Err(err).Str("client", c.id).Msg("failed to send ping; closing connection")
return
}
}
}
}
// sanitizeData recursively sanitizes data to replace NaN/Inf values with nil
func sanitizeData(data interface{}) interface{} {
// First, marshal to JSON to convert structs to maps
jsonBytes, err := json.Marshal(data)
if err != nil {
return data
}
var jsonData interface{}
if err := json.Unmarshal(jsonBytes, &jsonData); err != nil {
return data
}
return sanitizeValue(jsonData)
}
// sanitizeValue recursively sanitizes JSON-compatible values
func sanitizeValue(data interface{}) interface{} {
switch v := data.(type) {
case float64:
if math.IsNaN(v) || math.IsInf(v, 0) {
return nil
}
return v
case float32:
if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {
return nil
}
return v
case map[string]interface{}:
sanitized := make(map[string]interface{})
for k, val := range v {
sanitized[k] = sanitizeValue(val)
}
return sanitized
case []interface{}:
sanitized := make([]interface{}, len(v))
for i, val := range v {
sanitized[i] = sanitizeValue(val)
}
return sanitized
default:
// For all other types (string, bool, nil, etc.), return as-is
return v
}
}