Skip to content

Take control over your live stream video by running it yourself. Streaming + chat out of the box.

fix(activitypub): fixes #4372 where excessive AP resource use disrupts video delivery (#4552)

Gabe Kangasba7d92a19e16
diff --git a/activitypub/activitypub.go b/activitypub/activitypub.go
index 287ea7e526..8cf700848a 100644
--- a/activitypub/activitypub.go
+++ b/activitypub/activitypub.go
@@ -1,8 +1,6 @@
package activitypub
import (
- "math"
-
"github.com/owncast/owncast/activitypub/crypto"
"github.com/owncast/owncast/activitypub/inbox"
"github.com/owncast/owncast/activitypub/outbox"
@@ -36,14 +34,32 @@ func Start(datastore *data.Datastore) {
}
func getOutboundWorkerPoolSize() int {
+ // Use a reasonable fixed worker pool size instead of scaling with followers
+ // This prevents excessive resource usage when streamers have many followers
+ const (
+ minWorkers = 10 // Minimum workers for small instances
+ maxWorkers = 50 // Maximum workers to prevent resource exhaustion
+ defaultWorkers = 20 // Default for most instances
+ )
+
var followerCount int64
fc, err := persistence.GetFollowerCount()
if err != nil {
log.Errorln("Unable to get follower count", err)
- fc = 50 // Arbitrary fallback value.
+ return defaultWorkers
+ }
+ followerCount = fc
+
+ // Scale more conservatively: start with base workers, add 1 worker per 100 followers
+ // This gives a much more reasonable scaling than the previous followerCount * 5
+ workers := minWorkers + int(followerCount/100)
+
+ if workers > maxWorkers {
+ workers = maxWorkers
}
- followerCount = int64(math.Max(float64(fc), 50))
- return int(followerCount * 5)
+
+ log.Infof("Initializing ActivityPub outbound worker pool with %d workers for %d followers", workers, followerCount)
+ return workers
}
// SendLive will send a "Go Live" message to followers.
diff --git a/activitypub/workerpool/outbound.go b/activitypub/workerpool/outbound.go
index 0681869e0f..5494acc380 100644
--- a/activitypub/workerpool/outbound.go
+++ b/activitypub/workerpool/outbound.go
@@ -2,6 +2,8 @@ package workerpool
import (
"net/http"
+ "sync"
+ "time"
log "github.com/sirupsen/logrus"
)
@@ -13,10 +15,46 @@ type Job struct {
var queue chan Job
+// Circuit breaker backoff durations for exponential backoff.
+var circuitBreakerBackoffDurations = []time.Duration{
+ 1 * time.Minute, // 1st failure: 1 minute
+ 5 * time.Minute, // 2nd failure: 5 minutes
+ 15 * time.Minute, // 3rd failure: 15 minutes
+ 30 * time.Minute, // 4th failure: 30 minutes
+ 60 * time.Minute, // 5+ failures: 1 hour (max)
+}
+
+// httpClient is a configured HTTP client with timeouts and connection limits.
+var httpClient *http.Client
+
+// failedDomains tracks domains that are consistently failing with their failure count and backoff time.
+var (
+ failedDomains = make(map[string]*domainFailure)
+ failedDomainsMutex sync.RWMutex
+)
+
+type domainFailure struct {
+ count int
+ lastFailed time.Time
+ backoffUntil time.Time
+}
+
// InitOutboundWorkerPool starts n go routines that await ActivityPub jobs.
func InitOutboundWorkerPool(workerPoolSize int) {
queue = make(chan Job, workerPoolSize)
+ // Initialize HTTP client with conservative timeouts and connection limits
+ // to prevent resource exhaustion and hanging requests
+ httpClient = &http.Client{
+ Timeout: 15 * time.Second, // Reduced from 30s for faster failure detection
+ Transport: &http.Transport{
+ MaxIdleConns: 20, // Reduced from 100 to limit resource usage
+ MaxIdleConnsPerHost: 2, // Reduced from 10 to be more conservative
+ IdleConnTimeout: 10 * time.Second, // Reduced from 30s for faster cleanup
+ DisableKeepAlives: false,
+ },
+ }
+
// start workers
for i := 1; i <= workerPoolSize; i++ {
go worker(i, queue)
@@ -25,6 +63,12 @@ func InitOutboundWorkerPool(workerPoolSize int) {
// AddToOutboundQueue will queue up an outbound http request.
func AddToOutboundQueue(req *http.Request) {
+ // Check if domain should be skipped due to circuit breaker
+ if shouldSkipDomain(req.URL.Host) {
+ log.Debugf("Skipping request to %s due to circuit breaker", req.URL.Host)
+ return
+ }
+
select {
case queue <- Job{req}:
default:
@@ -40,20 +84,88 @@ func worker(workerID int, queue <-chan Job) {
for job := range queue {
if err := sendActivityPubMessageToInbox(job); err != nil {
log.Errorf("ActivityPub destination %s failed to send Error: %s", job.request.RequestURI, err)
+ recordDomainFailure(job.request.URL.Host)
+ } else {
+ // Reset domain failure count on success
+ resetDomainFailure(job.request.URL.Host)
}
log.Tracef("Done with ActivityPub destination %s using worker %d", job.request.RequestURI, workerID)
}
}
func sendActivityPubMessageToInbox(job Job) error {
- client := &http.Client{}
-
- resp, err := client.Do(job.request)
+ resp, err := httpClient.Do(job.request)
if err != nil {
return err
}
defer resp.Body.Close()
+ // Consider HTTP 4xx and 5xx as failures for circuit breaker purposes
+ if resp.StatusCode >= 400 {
+ return &httpError{statusCode: resp.StatusCode, message: resp.Status}
+ }
+
return nil
}
+
+// httpError represents an HTTP error response.
+type httpError struct {
+ statusCode int
+ message string
+}
+
+func (e *httpError) Error() string {
+ return e.message
+}
+
+// shouldSkipDomain checks if a domain should be skipped due to circuit breaker.
+func shouldSkipDomain(domain string) bool {
+ failedDomainsMutex.RLock()
+ defer failedDomainsMutex.RUnlock()
+
+ failure, exists := failedDomains[domain]
+ if !exists {
+ return false
+ }
+
+ // If we're still in backoff period, skip this domain
+ return time.Now().Before(failure.backoffUntil)
+}
+
+// recordDomainFailure records a failure for a domain and implements exponential backoff.
+func recordDomainFailure(domain string) {
+ failedDomainsMutex.Lock()
+ defer failedDomainsMutex.Unlock()
+
+ failure, exists := failedDomains[domain]
+ if !exists {
+ failure = &domainFailure{}
+ failedDomains[domain] = failure
+ }
+
+ failure.count++
+ failure.lastFailed = time.Now()
+
+ // Use exponential backoff with pre-defined durations
+ backoffIndex := failure.count - 1
+ if backoffIndex >= len(circuitBreakerBackoffDurations) {
+ backoffIndex = len(circuitBreakerBackoffDurations) - 1
+ }
+
+ backoffDuration := circuitBreakerBackoffDurations[backoffIndex]
+ failure.backoffUntil = time.Now().Add(backoffDuration)
+
+ log.Warnf("Domain %s failed %d times, backing off for %v", domain, failure.count, backoffDuration)
+}
+
+// resetDomainFailure resets the failure count for a domain on successful delivery.
+func resetDomainFailure(domain string) {
+ failedDomainsMutex.Lock()
+ defer failedDomainsMutex.Unlock()
+
+ if failure, exists := failedDomains[domain]; exists && failure.count > 0 {
+ log.Debugf("Resetting failure count for domain %s after successful delivery", domain)
+ delete(failedDomains, domain)
+ }
+}
diff --git a/activitypub/workerpool/outbound_test.go b/activitypub/workerpool/outbound_test.go
new file mode 100644
index 0000000000..a3660de3e3
--- /dev/null
+++ b/activitypub/workerpool/outbound_test.go
@@ -0,0 +1,127 @@
+package workerpool
+
+import (
+ "testing"
+ "time"
+)
+
+// resetCircuitBreakerForTesting clears all failed domain state for testing purposes.
+// This function provides test isolation by resetting the global circuit breaker state.
+func resetCircuitBreakerForTesting() {
+ failedDomainsMutex.Lock()
+ defer failedDomainsMutex.Unlock()
+ failedDomains = make(map[string]*domainFailure)
+}
+
+func TestCircuitBreaker(t *testing.T) {
+ // Ensure clean state before test
+ resetCircuitBreakerForTesting()
+ defer resetCircuitBreakerForTesting() // Clean up after test
+
+ testDomain := "failing.example.com"
+
+ // Initially, domain should not be skipped
+ if shouldSkipDomain(testDomain) {
+ t.Error("Domain should not be skipped initially")
+ }
+
+ // Record failures
+ recordDomainFailure(testDomain)
+ recordDomainFailure(testDomain)
+ recordDomainFailure(testDomain)
+
+ // Domain should now be skipped
+ if !shouldSkipDomain(testDomain) {
+ t.Error("Domain should be skipped after failures")
+ }
+
+ // After successful delivery, domain should be reset
+ resetDomainFailure(testDomain)
+ if shouldSkipDomain(testDomain) {
+ t.Error("Domain should not be skipped after reset")
+ }
+}
+
+func TestHTTPTimeouts(t *testing.T) {
+ // Ensure clean state before test
+ resetCircuitBreakerForTesting()
+ defer resetCircuitBreakerForTesting() // Clean up after test
+
+ // Initialize HTTP client
+ InitOutboundWorkerPool(1)
+
+ if httpClient == nil {
+ t.Error("HTTP client should be initialized")
+ }
+
+ if httpClient.Timeout != 15*time.Second {
+ t.Error("HTTP client should have 15 second timeout")
+ }
+}
+
+func TestWorkerPoolSizing(t *testing.T) {
+ // Ensure clean state before test
+ resetCircuitBreakerForTesting()
+ defer resetCircuitBreakerForTesting() // Clean up after test
+
+ // Test the queue sizing
+ InitOutboundWorkerPool(5)
+
+ if cap(queue) != 5 {
+ t.Errorf("Queue capacity should be 5, got %d", cap(queue))
+ }
+}
+
+func TestBackoffDurations(t *testing.T) {
+ // Test that backoff durations are properly configured
+ expectedDurations := []time.Duration{
+ 1 * time.Minute,
+ 5 * time.Minute,
+ 15 * time.Minute,
+ 30 * time.Minute,
+ 60 * time.Minute,
+ }
+
+ if len(circuitBreakerBackoffDurations) != len(expectedDurations) {
+ t.Errorf("Expected %d backoff durations, got %d", len(expectedDurations), len(circuitBreakerBackoffDurations))
+ }
+
+ for i, expected := range expectedDurations {
+ if circuitBreakerBackoffDurations[i] != expected {
+ t.Errorf("Backoff duration at index %d: expected %v, got %v", i, expected, circuitBreakerBackoffDurations[i])
+ }
+ }
+}
+
+func TestCircuitBreakerIsolation(t *testing.T) {
+ // Test that multiple tests don't interfere with each other
+ resetCircuitBreakerForTesting()
+ defer resetCircuitBreakerForTesting()
+
+ domain1 := "test1.example.com"
+ domain2 := "test2.example.com"
+
+ // Neither domain should be blocked initially
+ if shouldSkipDomain(domain1) || shouldSkipDomain(domain2) {
+ t.Error("Domains should not be blocked initially")
+ }
+
+ // Record failures for domain1 only
+ recordDomainFailure(domain1)
+ recordDomainFailure(domain1)
+ recordDomainFailure(domain1)
+
+ // Only domain1 should be blocked
+ if !shouldSkipDomain(domain1) {
+ t.Error("Domain1 should be blocked after failures")
+ }
+ if shouldSkipDomain(domain2) {
+ t.Error("Domain2 should not be blocked")
+ }
+
+ // Reset and verify clean state
+ resetCircuitBreakerForTesting()
+ if shouldSkipDomain(domain1) || shouldSkipDomain(domain2) {
+ t.Error("Both domains should be unblocked after reset")
+ }
+}