Skip to content

Take control over your live stream video by running it yourself. Streaming + chat out of the box.

fix(chat): synchronize client and stream state maps

Gabe Kangas4f5cf9c8b438
diff --git a/services/chat/chat.go b/services/chat/chat.go
index b027691dd6..d20396a799 100644
--- a/services/chat/chat.go
+++ b/services/chat/chat.go
@@ -83,8 +83,8 @@ func (s *Service) Start() error {
// GetClientsForUser will return chat connections that are owned by a specific user.
func (s *Service) GetClientsForUser(userID string) ([]*Client, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
+ s.mu.RLock()
+ defer s.mu.RUnlock()
clients := map[string][]*Client{}
@@ -101,21 +101,24 @@ func (s *Service) GetClientsForUser(userID string) ([]*Client, error) {
// FindClientByID will return a single connected client by ID.
func (s *Service) FindClientByID(clientID uint) (*Client, bool) {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
client, found := s.clients[clientID]
return client, found
}
// GetClients will return all the current chat clients connected.
func (s *Service) GetClients() []*Client {
- clients := []*Client{}
-
if s == nil {
- return clients
+ return []*Client{}
}
+ s.mu.RLock()
+ clients := make([]*Client, 0, len(s.clients))
for _, client := range s.clients {
clients = append(clients, client)
}
+ s.mu.RUnlock()
sort.Slice(clients, func(i, j int) bool {
return clients[i].ConnectedAt.Before(clients[j].ConnectedAt)
diff --git a/services/chat/chatclient.go b/services/chat/chatclient.go
index c6e440fc64..32f3e327d4 100644
--- a/services/chat/chatclient.go
+++ b/services/chat/chatclient.go
@@ -252,6 +252,22 @@ func (c *Client) sendPayload(payload interface{}) {
}
}
+func (c *Client) trySend(data []byte) bool {
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+
+ if c.send == nil {
+ return false
+ }
+
+ select {
+ case c.send <- data:
+ return true
+ default:
+ return false
+ }
+}
+
func (c *Client) sendAction(message string) {
clientMessage := events.ActionEvent{
MessageEvent: events.MessageEvent{
diff --git a/services/chat/server.go b/services/chat/server.go
index 8c1770f285..ccf788fe8b 100644
--- a/services/chat/server.go
+++ b/services/chat/server.go
@@ -158,8 +158,8 @@ func (s *Service) Addclient(conn *websocket.Conn, user *models.User, accessToken
s.mu.Lock()
{
// If there is a pending disconnect timer then clear it. A quick
- // reconnect before the part was sent is not a new join, so neither the
- // join broadcast nor the webhook should fire for it.
+ // reconnect before the part was sent is not a new join, so neither
+ // the join broadcast nor the webhook should fire for it.
if s.cancelUserPartedMessage(user.ID) {
isNewJoin = false
}
@@ -167,10 +167,10 @@ func (s *Service) Addclient(conn *websocket.Conn, user *models.User, accessToken
client.Id = s.seq
s.clients[client.Id] = client
s.seq++
+ logTotal := len(s.clients)
+ s.mu.Unlock()
+ log.Traceln("Adding client", client.Id, "total count:", logTotal)
}
- s.mu.Unlock()
-
- log.Traceln("Adding client", client.Id, "total count:", len(s.clients))
go client.writePump()
go client.readPump()
@@ -306,8 +306,10 @@ func (s *Service) HandleClientConnection(w http.ResponseWriter, r *http.Request)
log.Errorln("error determining if IP address is blocked: ", err)
}
- // Limit concurrent chat connections
- if uint64(len(s.clients)) >= s.maxSocketConnectionLimit {
+ s.mu.RLock()
+ clientCount := len(s.clients)
+ s.mu.RUnlock()
+ if uint64(clientCount) >= s.maxSocketConnectionLimit {
log.Warnln("rejecting incoming client connection as it exceeds the max client count of", s.maxSocketConnectionLimit)
_, _ = w.Write([]byte(events.ErrorMaxConnectionsExceeded))
return
@@ -383,17 +385,13 @@ func (s *Service) Broadcast(payload events.EventPayload) error {
return err
}
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- for _, client := range s.clients {
+ clients := s.GetClients()
+ for _, client := range clients {
if client == nil {
continue
}
- select {
- case client.send <- data:
- default:
+ if !client.trySend(data) {
go client.close()
}
}
@@ -403,13 +401,10 @@ func (s *Service) Broadcast(payload events.EventPayload) error {
// Send will send a single payload to a single connected client.
func (s *Service) Send(payload events.EventPayload, client *Client) {
- data, err := json.Marshal(payload)
- if err != nil {
- log.Errorln(err)
+ if client == nil {
return
}
-
- client.send <- data
+ client.sendPayload(payload)
}
// DisconnectClients will forcefully disconnect all clients belonging to a user by ID.
diff --git a/services/stream/stats.go b/services/stream/stats.go
index 406950ac3d..ee8cc6369e 100644
--- a/services/stream/stats.go
+++ b/services/stream/stats.go
@@ -13,7 +13,9 @@ import (
// periodic save + viewer-prune goroutines.
func (s *Service) setupStats() error {
saved := s.getSavedStats()
+ s.statsMu.Lock()
s.stats = &saved
+ s.statsMu.Unlock()
statsSaveTicker := time.NewTicker(1 * time.Minute)
go func() {
@@ -36,6 +38,14 @@ func (s *Service) setupStats() error {
// live HLS content. Returns false during the brief warm-up after an
// RTMP connection where HLS segments aren't ready yet.
func (s *Service) IsStreamConnected() bool {
+ waitTime := math.Max(float64(s.configRepository.GetStreamLatencyLevel().SecondsPerSegment)*3.0, 7)
+
+ s.statsMu.RLock()
+ defer s.statsMu.RUnlock()
+ return s.isStreamConnectedLocked(waitTime)
+}
+
+func (s *Service) isStreamConnectedLocked(waitTime float64) bool {
if s.stats == nil || !s.stats.StreamConnected {
return false
}
@@ -44,7 +54,6 @@ func (s *Service) IsStreamConnected() bool {
// connection and when HLS data is available. Account for that with
// an artificial buffer of a few segments.
timeSinceLastConnected := time.Since(s.stats.LastConnectTime.Time).Seconds()
- waitTime := math.Max(float64(s.configRepository.GetStreamLatencyLevel().SecondsPerSegment)*3.0, 7)
if timeSinceLastConnected < waitTime {
return false
}
@@ -59,20 +68,22 @@ func (s *Service) RemoveChatClient(clientID string) {
log.Trace("Removing the client:", clientID)
s.statsMu.Lock()
+ defer s.statsMu.Unlock()
+ if s.stats == nil {
+ return
+ }
delete(s.stats.ChatClients, clientID)
- s.statsMu.Unlock()
}
// SetViewerActive marks a viewer as currently watching and updates the
// session/overall peak counters. Silent no-op while no stream is live.
func (s *Service) SetViewerActive(viewer *models.Viewer) {
+ s.statsMu.Lock()
+ defer s.statsMu.Unlock()
if s.stats == nil || !s.stats.StreamConnected {
return
}
- s.statsMu.Lock()
- defer s.statsMu.Unlock()
-
// Asynchronously, optionally, fetch GeoIP info. The lookup can be slow,
// so it lands on the stored viewer later, under the same lock every
// other reader takes.
@@ -99,13 +110,12 @@ func (s *Service) SetViewerActive(viewer *models.Viewer) {
// The viewers are copied because the live map and the viewers in it are
// mutated by request handlers and the prune goroutine while callers read.
func (s *Service) GetActiveViewers() map[string]models.Viewer {
+ s.statsMu.RLock()
+ defer s.statsMu.RUnlock()
if s.stats == nil {
return nil
}
- s.statsMu.Lock()
- defer s.statsMu.Unlock()
-
viewers := make(map[string]models.Viewer, len(s.stats.Viewers))
for id, viewer := range s.stats.Viewers {
viewers[id] = *viewer
@@ -114,39 +124,52 @@ func (s *Service) GetActiveViewers() map[string]models.Viewer {
}
func (s *Service) pruneViewerCount() {
- viewers := make(map[string]*models.Viewer)
-
s.statsMu.Lock()
defer s.statsMu.Unlock()
+ if s.stats == nil {
+ return
+ }
+ viewers := make(map[string]*models.Viewer)
for viewerID, viewer := range s.stats.Viewers {
- if time.Since(s.stats.Viewers[viewerID].LastSeen) < activeViewerPurgeTimeout {
+ if time.Since(viewer.LastSeen) < activeViewerPurgeTimeout {
viewers[viewerID] = viewer
}
}
-
s.stats.Viewers = viewers
}
func (s *Service) saveStats() {
- if err := s.configRepository.SetPeakOverallViewerCount(s.stats.OverallMaxViewerCount); err != nil {
+ s.statsMu.RLock()
+ if s.stats == nil {
+ s.statsMu.RUnlock()
+ return
+ }
+ overallMax := s.stats.OverallMaxViewerCount
+ sessionMax := s.stats.SessionMaxViewerCount
+ streamConnected := s.stats.StreamConnected
+ lastConnect := s.stats.LastConnectTime
+ lastDisconnect := s.stats.LastDisconnectTime
+ s.statsMu.RUnlock()
+
+ if err := s.configRepository.SetPeakOverallViewerCount(overallMax); err != nil {
log.Errorln("error saving viewer count", err)
}
- if err := s.configRepository.SetPeakSessionViewerCount(s.stats.SessionMaxViewerCount); err != nil {
+ if err := s.configRepository.SetPeakSessionViewerCount(sessionMax); err != nil {
log.Errorln("error saving viewer count", err)
}
// Persist a recent live heartbeat while the stream is online so an unclean
// shutdown (SIGKILL, node reboot, container eviction) still leaves a useful
// "last live" timestamp on the next startup. A graceful disconnect below will
// overwrite this with the real end time.
- if s.stats.StreamConnected && s.stats.LastConnectTime != nil && s.stats.LastConnectTime.Valid {
+ if streamConnected && lastConnect != nil && lastConnect.Valid {
if err := s.configRepository.SetLastDisconnectTime(time.Now()); err != nil {
log.Errorln("error saving live heartbeat time", err)
}
return
}
- if s.stats.LastDisconnectTime != nil && s.stats.LastDisconnectTime.Valid {
- if err := s.configRepository.SetLastDisconnectTime(s.stats.LastDisconnectTime.Time); err != nil {
+ if lastDisconnect != nil && lastDisconnect.Valid {
+ if err := s.configRepository.SetLastDisconnectTime(lastDisconnect.Time); err != nil {
log.Errorln("error saving disconnect time", err)
}
}
diff --git a/services/stream/status.go b/services/stream/status.go
index 2421465492..b042c8b433 100644
--- a/services/stream/status.go
+++ b/services/stream/status.go
@@ -8,25 +8,35 @@ import (
// GetStatus returns a snapshot of the current stream state suitable for
// the public/admin status APIs.
func (s *Service) GetStatus() models.Status {
+ waitTime := float64(s.configRepository.GetStreamLatencyLevel().SecondsPerSegment) * 3.0
+ if waitTime < 7 {
+ waitTime = 7
+ }
+
+ s.statsMu.RLock()
if s.stats == nil {
+ s.statsMu.RUnlock()
return models.Status{}
}
+ online := s.isStreamConnectedLocked(waitTime)
viewerCount := 0
- if s.IsStreamConnected() {
+ if online {
viewerCount = len(s.stats.Viewers)
}
-
- return models.Status{
- Online: s.IsStreamConnected(),
+ status := models.Status{
+ Online: online,
ViewerCount: viewerCount,
OverallMaxViewerCount: s.stats.OverallMaxViewerCount,
SessionMaxViewerCount: s.stats.SessionMaxViewerCount,
LastDisconnectTime: s.stats.LastDisconnectTime,
LastConnectTime: s.stats.LastConnectTime,
VersionNumber: config.VersionNumber,
- StreamTitle: s.configRepository.GetStreamTitle(),
}
+ s.statsMu.RUnlock()
+
+ status.StreamTitle = s.configRepository.GetStreamTitle()
+ return status
}
// GetCurrentBroadcast returns the in-flight broadcast settings, or nil
diff --git a/services/stream/stream.go b/services/stream/stream.go
index ba2c7821b1..3f78ee394a 100644
--- a/services/stream/stream.go
+++ b/services/stream/stream.go
@@ -173,10 +173,12 @@ func (s *Service) StreamConnected(broadcast *models.CurrentBroadcast) {
// these reactions through StreamConnected.
func (s *Service) applyStreamOnline() {
now := utils.NullTime{Time: time.Now(), Valid: true}
+ s.statsMu.Lock()
s.stats.StreamConnected = true
s.stats.LastDisconnectTime = nil
s.stats.LastConnectTime = &now
s.stats.SessionMaxViewerCount = 0
+ s.statsMu.Unlock()
s.StopOfflineCleanupTimer()
s.startOnlineCleanupTimer()
@@ -222,13 +224,11 @@ func (s *Service) applyStreamOffline() {
_ = s.chat.SendSystemAction("The stream is ending.", true)
now := utils.NullTime{Time: time.Now(), Valid: true}
- if s.onlineTimerCancelFunc != nil {
- s.onlineTimerCancelFunc()
- }
-
+ s.statsMu.Lock()
s.stats.StreamConnected = false
s.stats.LastDisconnectTime = &now
s.stats.LastConnectTime = nil
+ s.statsMu.Unlock()
s.broadcaster = nil
// Stop the federated stream-ping ticker so we don't keep advertising