Take control over your live stream video by running it yourself. Streaming + chat out of the box.
HTML activitypubbroadcastingchatdecentralizedfederationfediversegolanghacktoberfesthlslivelivestreamowncastrtmpself-hostedstreaming-videovideo
ActivityPub background actor validation, update and cleanup process (#4765)
Gabe Kangase8308b64d47e
diff --git a/.github/workflows/activitypub-follower-validation-tests.yaml b/.github/workflows/activitypub-follower-validation-tests.yamlnew file mode 100644index 0000000000..efdcbd2d31--- /dev/null+++ b/.github/workflows/activitypub-follower-validation-tests.yaml@@ -0,0 +1,47 @@+name: ActivityPub Follower Validation Tests++on:+ push:+ paths:+ - "activitypub/**"+ - "config/**"+ - "main.go"+ - "test/automated/activitypub/**"+ - ".github/workflows/activitypub-follower-validation-tests.yaml"+ pull_request:+ paths:+ - "activitypub/**"+ - "config/**"+ - "main.go"+ - "test/automated/activitypub/**"+ - ".github/workflows/activitypub-follower-validation-tests.yaml"++jobs:+ follower-validation-test:+ runs-on: ubuntu-latest++ steps:+ - id: skip_check+ uses: fkirc/skip-duplicate-actions@v5+ with:+ concurrent_skipping: "same_content_newer"++ - name: Check out repository code+ uses: actions/checkout@v6++ - name: Set up Go+ uses: actions/setup-go@v6+ with:+ go-version: "^1.23"+ cache: true++ - name: Install dependencies+ run: |+ sudo apt-get update+ sudo apt-get install -y sqlite3 ffmpeg jq++ - name: Run follower validation test+ timeout-minutes: 15+ run: |+ cd test/automated/activitypub+ ./test-follower-validation.shdiff --git a/activitypub/activitypub.go b/activitypub/activitypub.goindex ef0f89c468..9b9c6c0377 100644--- a/activitypub/activitypub.go+++ b/activitypub/activitypub.go@@ -3,6 +3,7 @@ package activitypubimport ("github.com/owncast/owncast/activitypub/crypto""github.com/owncast/owncast/activitypub/inbox"+ "github.com/owncast/owncast/activitypub/jobs""github.com/owncast/owncast/activitypub/outbox""github.com/owncast/owncast/activitypub/persistence""github.com/owncast/owncast/activitypub/persistence/followersrepository"@@ -32,6 +33,9 @@ func Start(datastore *data.Datastore) {log.Errorln("Unable to get private key", err)}}++ // Start follower validation background job.+ jobs.StartFollowerValidationJob()}func getOutboundWorkerPoolSize() int {diff --git a/activitypub/jobs/followervalidation.go b/activitypub/jobs/followervalidation.gonew file mode 100644index 0000000000..325a7c2fc8--- /dev/null+++ b/activitypub/jobs/followervalidation.go@@ -0,0 +1,117 @@+package jobs++import (+ "time"++ "github.com/owncast/owncast/activitypub/persistence/followersrepository"+ "github.com/owncast/owncast/activitypub/resolvers"+ "github.com/owncast/owncast/config"+ "github.com/owncast/owncast/models"+ "github.com/owncast/owncast/persistence/configrepository"+ log "github.com/sirupsen/logrus"+)++const (+ // ValidationInterval is how often the validation job runs.+ ValidationInterval = 1 * time.Hour+ // FollowersPerRun is how many followers to validate per job run.+ FollowersPerRun = 5+ // FailureDurationThreshold is how long a follower must be unreachable before removal.+ FailureDurationThreshold = 7 * 24 * time.Hour // 7 days+ // DelayBetweenFollowers is the delay between validating individual followers.+ DelayBetweenFollowers = 2 * time.Second+)++// GetValidationInterval returns the configured validation interval, or the default.+func GetValidationInterval() time.Duration {+ if config.FollowerValidationInterval > 0 {+ return config.FollowerValidationInterval+ }+ return ValidationInterval+}++// StartFollowerValidationJob starts the background job that periodically validates followers.+func StartFollowerValidationJob() {+ interval := GetValidationInterval()+ ticker := time.NewTicker(interval)+ go func() {+ for range ticker.C {+ runFollowerValidation()+ }+ }()+ log.Debugf("Follower validation job scheduled with interval %v", interval)+}++func runFollowerValidation() {+ configRepo := configrepository.Get()+ if !configRepo.GetFederationEnabled() {+ return+ }++ followersRepo := followersrepository.Get()+ followers, err := followersRepo.GetFollowersToValidate(FollowersPerRun)+ if err != nil {+ log.Errorln("Failed to get followers for validation:", err)+ return+ }++ for _, follower := range followers {+ validateAndUpdateFollower(followersRepo, follower)+ time.Sleep(DelayBetweenFollowers)+ }+}++func validateAndUpdateFollower(repo followersrepository.FollowersRepository, follower models.Follower) {+ resolvedActor, err := resolvers.GetResolvedActorFromIRI(follower.ActorIRI)+ if err != nil {+ handleValidationFailure(repo, follower, err)+ return+ }++ // Success - clear failure timestamp and update data+ if err := repo.UpdateFollowerValidationSuccess(follower.ActorIRI); err != nil {+ log.Errorln("Failed to update validation success:", err)+ }++ // Update follower data+ if err := repo.Update(+ resolvedActor.ActorIriString(),+ resolvedActor.InboxString(),+ resolvedActor.SharedInboxString(),+ resolvedActor.Name,+ resolvedActor.FullUsername,+ resolvedActor.ImageString(),+ ); err != nil {+ log.Errorln("Failed to update follower data:", err)+ }+}++func handleValidationFailure(repo followersrepository.FollowersRepository, follower models.Follower, resolveErr error) {+ log.Debugf("Follower validation failed for %s: %v", follower.ActorIRI, resolveErr)++ // Check removal eligibility BEFORE updating failure timestamp.+ // We use the existing FirstValidationFailureAt from the database to determine+ // if this follower has been failing long enough to be removed.+ // If FirstValidationFailureAt is not set, this is a new failure and we just record it.+ shouldRemove := false+ if follower.FirstValidationFailureAt.Valid {+ failureDuration := time.Since(follower.FirstValidationFailureAt.Time)+ shouldRemove = failureDuration >= FailureDurationThreshold+ }++ // Update failure timestamp (sets first_validation_failure_at if not already set)+ if err := repo.UpdateFollowerValidationFailure(follower.ActorIRI); err != nil {+ log.Errorln("Failed to update validation failure:", err)+ return+ }++ // Remove follower if they've exceeded the failure threshold+ if shouldRemove {+ failureDuration := time.Since(follower.FirstValidationFailureAt.Time)+ log.Infof("Removing follower %s after %v of consecutive failures",+ follower.ActorIRI, failureDuration.Round(time.Hour))+ if err := repo.RemoveByIRI(follower.ActorIRI); err != nil {+ log.Errorln("Failed to remove invalid follower:", err)+ }+ }+}diff --git a/activitypub/jobs/followervalidation_test.go b/activitypub/jobs/followervalidation_test.gonew file mode 100644index 0000000000..0bfe9d0bb8--- /dev/null+++ b/activitypub/jobs/followervalidation_test.go@@ -0,0 +1,351 @@+package jobs++import (+ "net/url"+ "os"+ "testing"+ "time"++ "github.com/go-fed/activity/streams"+ "github.com/owncast/owncast/activitypub/apmodels"+ "github.com/owncast/owncast/activitypub/persistence"+ "github.com/owncast/owncast/activitypub/persistence/followersrepository"+ "github.com/owncast/owncast/core/data"+ "github.com/owncast/owncast/utils"+)++func TestMain(m *testing.M) {+ setup()+ code := m.Run()+ os.Exit(code)+}++var datastore *data.Datastore++func setup() {+ resetTestDatabase()+}++// resetTestDatabase initializes a fresh in-memory database for testing.+func resetTestDatabase() {+ data.SetupPersistence(":memory:")+ datastore = data.GetDatastore()+ persistence.Setup(datastore)+}++// setupTestWithRepo resets the database and returns a new repository instance.+func setupTestWithRepo(t *testing.T) followersrepository.FollowersRepository {+ t.Helper()+ resetTestDatabase()+ return followersrepository.New(datastore)+}++func createTestFollower(repo followersrepository.FollowersRepository, iri, inbox, name, username string) {+ actorIRI, _ := url.Parse(iri)+ inboxURL, _ := url.Parse(inbox)+ requestIRI, _ := url.Parse("https://fake.server/follow/request")+ fakeRequest := streams.NewActivityStreamsFollow()++ repo.Add(apmodels.ActivityPubActor{+ ActorIri: actorIRI,+ Inbox: inboxURL,+ Name: name,+ Username: username,+ FullUsername: username + "@fake.server",+ FollowRequestIri: requestIRI,+ RequestObject: fakeRequest,+ }, true)+}++func TestGetFollowersToValidate(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create some test followers+ for i := 0; i < 10; i++ {+ user, _ := utils.GenerateRandomString(10)+ createTestFollower(repo, "https://fake.server/user/"+user, "https://fake.server/user/"+user+"/inbox", user, user)+ }++ // Get followers to validate+ followers, err := repo.GetFollowersToValidate(5)+ if err != nil {+ t.Fatalf("Error getting followers to validate: %s", err)+ }++ if len(followers) != 5 {+ t.Errorf("Expected 5 followers to validate, got %d", len(followers))+ }+}++func TestUpdateFollowerValidationSuccess(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create a test follower+ testIRI := "https://fake.server/user/testuser"+ createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")++ // Mark validation as successful+ err := repo.UpdateFollowerValidationSuccess(testIRI)+ if err != nil {+ t.Fatalf("Error updating follower validation success: %s", err)+ }++ // Verify the follower was updated+ followers, err := repo.GetFollowersToValidate(10)+ if err != nil {+ t.Fatalf("Error getting followers: %s", err)+ }++ // After validation success, FirstValidationFailureAt should be NULL/invalid+ for _, f := range followers {+ if f.ActorIRI == testIRI {+ if f.FirstValidationFailureAt.Valid {+ t.Error("Expected FirstValidationFailureAt to be NULL after success")+ }+ }+ }+}++func TestUpdateFollowerValidationFailure(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create a test follower+ testIRI := "https://fake.server/user/testuser"+ createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")++ // Mark validation as failed+ err := repo.UpdateFollowerValidationFailure(testIRI)+ if err != nil {+ t.Fatalf("Error updating follower validation failure: %s", err)+ }++ // Verify the FirstValidationFailureAt is set+ followers, err := repo.GetFollowersToValidate(10)+ if err != nil {+ t.Fatalf("Error getting followers: %s", err)+ }++ found := false+ for _, f := range followers {+ if f.ActorIRI == testIRI {+ found = true+ if !f.FirstValidationFailureAt.Valid {+ t.Error("Expected FirstValidationFailureAt to be set after failure")+ }+ }+ }++ if !found {+ t.Error("Test follower not found in results")+ }+}++func TestValidationFailureClearedOnSuccess(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create a test follower+ testIRI := "https://fake.server/user/testuser"+ createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")++ // Mark validation as failed first+ err := repo.UpdateFollowerValidationFailure(testIRI)+ if err != nil {+ t.Fatalf("Error updating follower validation failure: %s", err)+ }++ // Then mark as successful+ err = repo.UpdateFollowerValidationSuccess(testIRI)+ if err != nil {+ t.Fatalf("Error updating follower validation success: %s", err)+ }++ // Verify FirstValidationFailureAt is cleared+ followers, err := repo.GetFollowersToValidate(10)+ if err != nil {+ t.Fatalf("Error getting followers: %s", err)+ }++ for _, f := range followers {+ if f.ActorIRI == testIRI {+ if f.FirstValidationFailureAt.Valid {+ t.Error("Expected FirstValidationFailureAt to be NULL after success")+ }+ }+ }+}++func TestRemoveByIRI(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create a test follower+ testIRI := "https://fake.server/user/testuser"+ createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")++ // Verify follower exists+ count, err := repo.GetCount()+ if err != nil {+ t.Fatalf("Error getting count: %s", err)+ }+ if count != 1 {+ t.Errorf("Expected 1 follower, got %d", count)+ }++ // Remove the follower+ err = repo.RemoveByIRI(testIRI)+ if err != nil {+ t.Fatalf("Error removing follower: %s", err)+ }++ // Verify follower is removed+ count, err = repo.GetCount()+ if err != nil {+ t.Fatalf("Error getting count: %s", err)+ }+ if count != 0 {+ t.Errorf("Expected 0 followers after removal, got %d", count)+ }+}++func TestFailureDurationThresholdLogic(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create a test follower+ testIRI := "https://fake.server/user/testfailure"+ createTestFollower(repo, testIRI, "https://fake.server/user/testfailure/inbox", "Test User", "testfailure")++ // Set first_validation_failure_at to 8 days ago (past threshold)+ eightDaysAgo := time.Now().Add(-8 * 24 * time.Hour)+ _, err := datastore.DB.Exec(+ "UPDATE ap_followers SET first_validation_failure_at = ? WHERE iri = ?",+ eightDaysAgo, testIRI,+ )+ if err != nil {+ t.Fatalf("Error setting first_validation_failure_at: %s", err)+ }++ // Fetch the follower+ followers, err := repo.GetFollowersToValidate(1)+ if err != nil {+ t.Fatalf("Error getting followers: %s", err)+ }++ if len(followers) != 1 {+ t.Fatalf("Expected 1 follower, got %d", len(followers))+ }++ // Verify the failure duration is calculated correctly+ failureDuration := time.Since(followers[0].FirstValidationFailureAt.Time)+ if failureDuration < FailureDurationThreshold {+ t.Errorf("Expected failure duration (%v) to be >= threshold (%v)", failureDuration, FailureDurationThreshold)+ }++ // Test removal - this directly tests the removal logic without network calls+ err = repo.RemoveByIRI(testIRI)+ if err != nil {+ t.Fatalf("Error removing follower: %s", err)+ }++ // Verify the follower was removed+ count, err := repo.GetCount()+ if err != nil {+ t.Fatalf("Error getting count: %s", err)+ }+ if count != 0 {+ t.Errorf("Expected follower to be removed, but count is %d", count)+ }+}++func TestFailureNotRemovedBeforeThreshold(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create a test follower+ testIRI := "https://fake.server/user/testnotremoved"+ createTestFollower(repo, testIRI, "https://fake.server/user/testnotremoved/inbox", "Test User", "testnotremoved")++ // Set first_validation_failure_at to 1 day ago (not past threshold)+ oneDayAgo := time.Now().Add(-1 * 24 * time.Hour)+ _, err := datastore.DB.Exec(+ "UPDATE ap_followers SET first_validation_failure_at = ? WHERE iri = ?",+ oneDayAgo, testIRI,+ )+ if err != nil {+ t.Fatalf("Error setting first_validation_failure_at: %s", err)+ }++ // Fetch the follower+ followers, err := repo.GetFollowersToValidate(1)+ if err != nil {+ t.Fatalf("Error getting followers: %s", err)+ }++ if len(followers) != 1 {+ t.Fatalf("Expected 1 follower, got %d", len(followers))+ }++ // Verify the failure duration is below threshold+ failureDuration := time.Since(followers[0].FirstValidationFailureAt.Time)+ if failureDuration >= FailureDurationThreshold {+ t.Errorf("Expected failure duration (%v) to be < threshold (%v)", failureDuration, FailureDurationThreshold)+ }++ // Follower should NOT be removed yet+ count, err := repo.GetCount()+ if err != nil {+ t.Fatalf("Error getting count: %s", err)+ }+ if count != 1 {+ t.Errorf("Expected follower to NOT be removed yet, but count is %d", count)+ }+}++func TestFollowersOrderedByOldestValidatedFirst(t *testing.T) {+ repo := setupTestWithRepo(t)++ // Create followers+ iri1 := "https://fake.server/user/first"+ iri2 := "https://fake.server/user/second"+ iri3 := "https://fake.server/user/third"++ createTestFollower(repo, iri1, "https://fake.server/user/first/inbox", "First", "first")+ createTestFollower(repo, iri2, "https://fake.server/user/second/inbox", "Second", "second")+ createTestFollower(repo, iri3, "https://fake.server/user/third/inbox", "Third", "third")++ // Set different last_validated_at times+ now := time.Now()+ _, err := datastore.DB.Exec("UPDATE ap_followers SET last_validated_at = ? WHERE iri = ?",+ now.Add(-3*time.Hour), iri1)+ if err != nil {+ t.Fatalf("Error updating: %s", err)+ }+ _, err = datastore.DB.Exec("UPDATE ap_followers SET last_validated_at = ? WHERE iri = ?",+ now.Add(-1*time.Hour), iri2)+ if err != nil {+ t.Fatalf("Error updating: %s", err)+ }+ // iri3 has NULL last_validated_at (never validated)++ // Get followers - should return in order: iri3 (NULL), iri1 (oldest), iri2 (newest)+ followers, err := repo.GetFollowersToValidate(3)+ if err != nil {+ t.Fatalf("Error getting followers: %s", err)+ }++ if len(followers) != 3 {+ t.Fatalf("Expected 3 followers, got %d", len(followers))+ }++ // NULL values should come first (NULLS FIRST in query)+ if followers[0].ActorIRI != iri3 {+ t.Errorf("Expected first follower to be %s (never validated), got %s", iri3, followers[0].ActorIRI)+ }++ // Then oldest validated+ if followers[1].ActorIRI != iri1 {+ t.Errorf("Expected second follower to be %s (oldest validated), got %s", iri1, followers[1].ActorIRI)+ }++ // Then newest validated+ if followers[2].ActorIRI != iri2 {+ t.Errorf("Expected third follower to be %s (newest validated), got %s", iri2, followers[2].ActorIRI)+ }+}diff --git a/activitypub/persistence/followers.go b/activitypub/persistence/followers.goindex deb22d330b..1a5cebdebd 100644--- a/activitypub/persistence/followers.go+++ b/activitypub/persistence/followers.go@@ -19,6 +19,8 @@ func createFederationFollowersTable() {"approved_at" TIMESTAMP,"disabled_at" TIMESTAMP,"request_object" BLOB,+ "last_validated_at" TIMESTAMP,+ "first_validation_failure_at" TIMESTAMP,PRIMARY KEY (iri));`_datastore.MustExec(createTableSQL)_datastore.MustExec(`CREATE INDEX IF NOT EXISTS idx_iri ON ap_followers (iri);`)diff --git a/activitypub/persistence/followersrepository/followersrepository.go b/activitypub/persistence/followersrepository/followersrepository.goindex d72019795a..5ba6113e85 100644--- a/activitypub/persistence/followersrepository/followersrepository.go+++ b/activitypub/persistence/followersrepository/followersrepository.go@@ -42,6 +42,14 @@ type FollowersRepository interface {BlockOrReject(iri string) error// Update updates the details of a stored follower.Update(actorIRI string, inbox string, sharedInbox string, name string, username string, image string) error+ // GetFollowersToValidate returns followers needing validation, ordered by oldest validated first.+ GetFollowersToValidate(limit int) ([]models.Follower, error)+ // UpdateFollowerValidationSuccess marks a follower as successfully validated and clears failure timestamp.+ UpdateFollowerValidationSuccess(iri string) error+ // UpdateFollowerValidationFailure marks a validation failure, setting first failure time if not already set.+ UpdateFollowerValidationFailure(iri string) error+ // RemoveByIRI removes a follower directly by IRI string.+ RemoveByIRI(iri string) error}// SqlFollowersRepository is the SQL-based implementation of FollowersRepository.@@ -367,3 +375,98 @@ func (r *SqlFollowersRepository) removeFollow(actor *url.URL) error {return tx.Commit()}++// GetFollowersToValidate returns followers needing validation, ordered by oldest validated first.+func (r *SqlFollowersRepository) GetFollowersToValidate(limit int) ([]models.Follower, error) {+ ctx := context.Background()+ followersResult, err := r.datastore.GetQueries().GetFollowersToValidate(ctx, utils.SafeIntToInt32(limit))+ if err != nil {+ return nil, err+ }++ followers := make([]models.Follower, 0)++ for _, row := range followersResult {+ singleFollower := models.Follower{+ Name: row.Name.String,+ Username: row.Username,+ Image: row.Image.String,+ ActorIRI: row.Iri,+ Inbox: row.Inbox,+ SharedInbox: row.SharedInbox.String,+ FirstValidationFailureAt: utils.NullTime(row.FirstValidationFailureAt),+ }++ followers = append(followers, singleFollower)+ }++ return followers, nil+}++// UpdateFollowerValidationSuccess marks a follower as successfully validated and clears failure timestamp.+func (r *SqlFollowersRepository) UpdateFollowerValidationSuccess(iri string) error {+ r.datastore.DbLock.Lock()+ defer r.datastore.DbLock.Unlock()++ ctx := context.Background()+ tx, err := r.datastore.DB.Begin()+ if err != nil {+ return errors.Wrap(err, "error beginning transaction")+ }+ defer func() {+ _ = tx.Rollback()+ }()++ if err := r.datastore.GetQueries().WithTx(tx).UpdateFollowerValidationSuccess(ctx, db.UpdateFollowerValidationSuccessParams{+ LastValidatedAt: sql.NullTime{Time: time.Now(), Valid: true},+ Iri: iri,+ }); err != nil {+ return errors.Wrap(err, "error updating follower validation success")+ }++ return tx.Commit()+}++// UpdateFollowerValidationFailure marks a validation failure, setting first failure time if not already set.+func (r *SqlFollowersRepository) UpdateFollowerValidationFailure(iri string) error {+ r.datastore.DbLock.Lock()+ defer r.datastore.DbLock.Unlock()++ ctx := context.Background()+ tx, err := r.datastore.DB.Begin()+ if err != nil {+ return errors.Wrap(err, "error beginning transaction")+ }+ defer func() {+ _ = tx.Rollback()+ }()++ if err := r.datastore.GetQueries().WithTx(tx).UpdateFollowerValidationFailure(ctx, db.UpdateFollowerValidationFailureParams{+ LastValidatedAt: sql.NullTime{Time: time.Now(), Valid: true},+ Iri: iri,+ }); err != nil {+ return errors.Wrap(err, "error updating follower validation failure")+ }++ return tx.Commit()+}++// RemoveByIRI removes a follower directly by IRI string.+func (r *SqlFollowersRepository) RemoveByIRI(iri string) error {+ r.datastore.DbLock.Lock()+ defer r.datastore.DbLock.Unlock()++ tx, err := r.datastore.DB.Begin()+ if err != nil {+ return err+ }+ defer func() {+ _ = tx.Rollback()+ }()++ if err := r.datastore.GetQueries().WithTx(tx).RemoveFollowerByIRI(context.Background(), iri); err != nil {+ return err+ }++ return tx.Commit()+}diff --git a/config/config.go b/config/config.goindex d4f590382a..4e07c448f2 100644--- a/config/config.go+++ b/config/config.go@@ -43,6 +43,10 @@ var EnableAutoUpdate = false// A temporary stream key that can be set via the command line.var TemporaryStreamKey = ""+// FollowerValidationInterval is how often the follower validation job runs.+// Defaults to 0 which means use the default (1 hour).+var FollowerValidationInterval time.Duration = 0+// GetCommit will return an identifier used for identifying the point in time this build took place.func GetCommit() string {if GitCommit == "" {diff --git a/core/data/data.go b/core/data/data.goindex 765388d4f8..5063f21e92 100644--- a/core/data/data.go+++ b/core/data/data.go@@ -19,7 +19,7 @@ import ()const (- schemaVersion = 8+ schemaVersion = 9)var (diff --git a/db/models.go b/db/models.goindex 80c2288e0a..63c1373409 100644--- a/db/models.go+++ b/db/models.go@@ -18,17 +18,19 @@ type ApAcceptedActivity struct {}type ApFollower struct {- Iri string- Inbox string- SharedInbox sql.NullString- Name sql.NullString- Username string- Image sql.NullString- Request string- RequestObject []byte- CreatedAt sql.NullTime- ApprovedAt sql.NullTime- DisabledAt sql.NullTime+ Iri string+ Inbox string+ SharedInbox sql.NullString+ Name sql.NullString+ Username string+ Image sql.NullString+ Request string+ RequestObject []byte+ CreatedAt sql.NullTime+ ApprovedAt sql.NullTime+ DisabledAt sql.NullTime+ LastValidatedAt sql.NullTime+ FirstValidationFailureAt sql.NullTime}type ApOutbox struct {diff --git a/db/query.sql b/db/query.sqlindex b1a1e5ce20..a816781d43 100644--- a/db/query.sql+++ b/db/query.sql@@ -57,6 +57,23 @@ SELECT count(*) FROM ap_accepted_activities WHERE iri = $1 AND actor = $2 AND TY-- name: UpdateFollowerByIRI :execUPDATE ap_followers SET inbox = $1, shared_inbox = $2, name = $3, username = $4, image = $5 WHERE iri = $6;+-- name: GetFollowersToValidate :many+SELECT iri, inbox, shared_inbox, name, username, image, first_validation_failure_at+FROM ap_followers+WHERE approved_at IS NOT NULL AND disabled_at IS NULL+ORDER BY last_validated_at ASC NULLS FIRST+LIMIT $1;++-- name: UpdateFollowerValidationSuccess :exec+UPDATE ap_followers+SET last_validated_at = $1, first_validation_failure_at = NULL+WHERE iri = $2;++-- name: UpdateFollowerValidationFailure :exec+UPDATE ap_followers+SET last_validated_at = $1, first_validation_failure_at = COALESCE(first_validation_failure_at, $1)+WHERE iri = $2;+-- name: GetUniqueDeliveryInboxes :manySELECT COALESCE(shared_inbox, inbox) as delivery_inbox FROM ap_followers WHERE approved_at is not null GROUP BY delivery_inbox;diff --git a/db/query.sql.go b/db/query.sql.goindex 54d3839bf5..2bd970b8c8 100644--- a/db/query.sql.go+++ b/db/query.sql.go@@ -306,9 +306,23 @@ const getFollowerByIRI = `-- name: GetFollowerByIRI :oneSELECT iri, inbox, shared_inbox, name, username, image, request, request_object, created_at, approved_at, disabled_at FROM ap_followers WHERE iri = $1`-func (q *Queries) GetFollowerByIRI(ctx context.Context, iri string) (ApFollower, error) {+type GetFollowerByIRIRow struct {+ Iri string+ Inbox string+ SharedInbox sql.NullString+ Name sql.NullString+ Username string+ Image sql.NullString+ Request string+ RequestObject []byte+ CreatedAt sql.NullTime+ ApprovedAt sql.NullTime+ DisabledAt sql.NullTime+}++func (q *Queries) GetFollowerByIRI(ctx context.Context, iri string) (GetFollowerByIRIRow, error) {row := q.db.QueryRowContext(ctx, getFollowerByIRI, iri)- var i ApFollower+ var i GetFollowerByIRIRowerr := row.Scan(&i.Iri,&i.Inbox,@@ -340,6 +354,55 @@ func (q *Queries) GetFollowerCount(ctx context.Context) (int64, error) {return count, err}+const getFollowersToValidate = `-- name: GetFollowersToValidate :many+SELECT iri, inbox, shared_inbox, name, username, image, first_validation_failure_at+FROM ap_followers+WHERE approved_at IS NOT NULL AND disabled_at IS NULL+ORDER BY last_validated_at ASC NULLS FIRST+LIMIT $1+`++type GetFollowersToValidateRow struct {+ Iri string+ Inbox string+ SharedInbox sql.NullString+ Name sql.NullString+ Username string+ Image sql.NullString+ FirstValidationFailureAt sql.NullTime+}++func (q *Queries) GetFollowersToValidate(ctx context.Context, limit int32) ([]GetFollowersToValidateRow, error) {+ rows, err := q.db.QueryContext(ctx, getFollowersToValidate, limit)+ if err != nil {+ return nil, err+ }+ defer rows.Close()+ var items []GetFollowersToValidateRow+ for rows.Next() {+ var i GetFollowersToValidateRow+ if err := rows.Scan(+ &i.Iri,+ &i.Inbox,+ &i.SharedInbox,+ &i.Name,+ &i.Username,+ &i.Image,+ &i.FirstValidationFailureAt,+ ); err != nil {+ return nil, err+ }+ items = append(items, i)+ }+ if err := rows.Close(); err != nil {+ return nil, err+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return items, nil+}+const getIPAddressBans = `-- name: GetIPAddressBans :manySELECT ip_address, notes, created_at FROM ip_bans`@@ -815,3 +878,35 @@ func (q *Queries) UpdateFollowerByIRI(ctx context.Context, arg UpdateFollowerByI)return err}++const updateFollowerValidationFailure = `-- name: UpdateFollowerValidationFailure :exec+UPDATE ap_followers+SET last_validated_at = $1, first_validation_failure_at = COALESCE(first_validation_failure_at, $1)+WHERE iri = $2+`++type UpdateFollowerValidationFailureParams struct {+ LastValidatedAt sql.NullTime+ Iri string+}++func (q *Queries) UpdateFollowerValidationFailure(ctx context.Context, arg UpdateFollowerValidationFailureParams) error {+ _, err := q.db.ExecContext(ctx, updateFollowerValidationFailure, arg.LastValidatedAt, arg.Iri)+ return err+}++const updateFollowerValidationSuccess = `-- name: UpdateFollowerValidationSuccess :exec+UPDATE ap_followers+SET last_validated_at = $1, first_validation_failure_at = NULL+WHERE iri = $2+`++type UpdateFollowerValidationSuccessParams struct {+ LastValidatedAt sql.NullTime+ Iri string+}++func (q *Queries) UpdateFollowerValidationSuccess(ctx context.Context, arg UpdateFollowerValidationSuccessParams) error {+ _, err := q.db.ExecContext(ctx, updateFollowerValidationSuccess, arg.LastValidatedAt, arg.Iri)+ return err+}diff --git a/db/schema.sql b/db/schema.sqlindex e0de4700ee..083b173481 100644--- a/db/schema.sql+++ b/db/schema.sql@@ -13,6 +13,8 @@ CREATE TABLE IF NOT EXISTS ap_followers ("created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP,"approved_at" TIMESTAMP,"disabled_at" TIMESTAMP,+ "last_validated_at" TIMESTAMP,+ "first_validation_failure_at" TIMESTAMP,PRIMARY KEY (iri));CREATE INDEX iri_index ON ap_followers (iri);CREATE INDEX approved_at_index ON ap_followers (approved_at);diff --git a/main.go b/main.goindex 562ebd19a4..c391382c88 100644--- a/main.go+++ b/main.go@@ -4,6 +4,7 @@ import ("flag""os""strconv"+ "time""github.com/owncast/owncast/logging""github.com/owncast/owncast/persistence/configrepository"@@ -18,17 +19,18 @@ import ()var (- dbFile = flag.String("database", "", "Path to the database file.")- logDirectory = flag.String("logdir", "", "Directory where logs will be written to")- backupDirectory = flag.String("backupdir", "", "Directory where backups will be written to")- enableDebugOptions = flag.Bool("enableDebugFeatures", false, "Enable additional debugging options.")- enableVerboseLogging = flag.Bool("enableVerboseLogging", false, "Enable additional logging.")- restoreDatabaseFile = flag.String("restoreDatabase", "", "Restore an Owncast database backup")- newAdminPassword = flag.String("adminpassword", "", "Set your admin password")- newStreamKey = flag.String("streamkey", "", "Set a temporary stream key for this session")- webServerPortOverride = flag.String("webserverport", "", "Force the web server to listen on a specific port")- webServerIPOverride = flag.String("webserverip", "", "Force web server to listen on this IP address")- rtmpPortOverride = flag.Int("rtmpport", 0, "Set listen port for the RTMP server")+ dbFile = flag.String("database", "", "Path to the database file.")+ logDirectory = flag.String("logdir", "", "Directory where logs will be written to")+ backupDirectory = flag.String("backupdir", "", "Directory where backups will be written to")+ enableDebugOptions = flag.Bool("enableDebugFeatures", false, "Enable additional debugging options.")+ enableVerboseLogging = flag.Bool("enableVerboseLogging", false, "Enable additional logging.")+ restoreDatabaseFile = flag.String("restoreDatabase", "", "Restore an Owncast database backup")+ newAdminPassword = flag.String("adminpassword", "", "Set your admin password")+ newStreamKey = flag.String("streamkey", "", "Set a temporary stream key for this session")+ webServerPortOverride = flag.String("webserverport", "", "Force the web server to listen on a specific port")+ webServerIPOverride = flag.String("webserverip", "", "Force web server to listen on this IP address")+ rtmpPortOverride = flag.Int("rtmpport", 0, "Set listen port for the RTMP server")+ followerValidationIntervalSecs = flag.Int("followervalidationinterval", 0, "Set follower validation interval in seconds"))// nolint:cyclop@@ -159,6 +161,12 @@ func handleCommandLineFlags() {log.Errorln(err)}}++ // Set the follower validation interval+ if *followerValidationIntervalSecs > 0 {+ config.FollowerValidationInterval = time.Duration(*followerValidationIntervalSecs) * time.Second+ log.Printf("Follower validation interval set to %v", config.FollowerValidationInterval)+ }}func configureLogging(enableDebugFeatures bool, enableVerboseLogging bool) {diff --git a/models/follower.go b/models/follower.goindex d7b60bd43f..99370fdc8f 100644--- a/models/follower.go+++ b/models/follower.go@@ -20,4 +20,6 @@ type Follower struct {Timestamp utils.NullTime `json:"timestamp,omitempty"`// DisabledAt is when this follower was rejected or disabled.DisabledAt utils.NullTime `json:"disabledAt,omitempty"`+ // FirstValidationFailureAt is when consecutive validation failures started.+ FirstValidationFailureAt utils.NullTime `json:"-"`}diff --git a/persistence/tables/migrations.go b/persistence/tables/migrations.goindex 0443faa5c8..7df284bf64 100644--- a/persistence/tables/migrations.go+++ b/persistence/tables/migrations.go@@ -35,6 +35,8 @@ func MigrateDatabaseSchema(db *sql.DB, from, to int) error {migrateToSchema7(db)case 7:migrateToSchema8(db)+ case 8:+ migrateToSchema9(db)default:log.Fatalln("missing database migration step")}@@ -48,6 +50,21 @@ func MigrateDatabaseSchema(db *sql.DB, from, to int) error {return nil}+func migrateToSchema9(db *sql.DB) {+ // Add columns for tracking follower validation status.+ // last_validated_at: when the follower was last successfully validated+ // first_validation_failure_at: when consecutive validation failures started (for removal threshold)+ alterStmts := []string{+ "ALTER TABLE ap_followers ADD COLUMN last_validated_at TIMESTAMP",+ "ALTER TABLE ap_followers ADD COLUMN first_validation_failure_at TIMESTAMP",+ }+ for _, stmt := range alterStmts {+ if _, err := db.Exec(stmt); err != nil {+ log.Warnln("Migration statement may have already been applied:", err)+ }+ }+}+func migrateToSchema8(db *sql.DB) {// Add shared_inbox column to ap_followers for ActivityPub shared inbox support.// This allows delivering messages to a server's shared inbox instead of eachdiff --git a/test/automated/activitypub/test-follower-validation.sh b/test/automated/activitypub/test-follower-validation.shnew file mode 100755index 0000000000..e52e24350e--- /dev/null+++ b/test/automated/activitypub/test-follower-validation.sh@@ -0,0 +1,519 @@+#!/bin/bash+# shellcheck disable=SC2317 # cleanup() is invoked via trap, not direct call++# Follower Validation Test+#+# This test verifies that the follower validation job correctly:+# 1. Removes invalid/fake followers that have been failing for > 7 days+# 2. Keeps valid real followers+#+# The test:+# 1. Builds and starts Owncast with a 20-second validation interval+# 2. Configures federation via admin API+# 3. Inserts 5 fake followers and 5 real followers directly into the database+# 4. Sets first_validation_failure_at to 8+ days ago for fake followers+# 5. Waits for the validation job to run+# 6. Verifies fake followers were removed and real followers remain++set -e++_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO_ROOT="$(git rev-parse --show-toplevel)"++# Configuration+OWNCAST_PORT="${OWNCAST_PORT:-8080}"+ADMIN_USER="admin"+ADMIN_PASS="abc123"+FEDERATION_USERNAME="streamer"+VALIDATION_INTERVAL=20 # seconds+MAX_TEST_DURATION=600 # Maximum test duration in seconds (10 minutes)+TEST_START_TIME=""++# Directories+TEMP_DIR=""+OWNCAST_DB=""++# PIDs+OWNCAST_PID=""++# Colors+RED='\033[0;31m'+GREEN='\033[0;32m'+YELLOW='\033[1;33m'+CYAN='\033[0;36m'+NC='\033[0m'++log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }+log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }+log_error() { echo -e "${RED}[ERROR]${NC} $1"; }+log_test() { echo -e "${CYAN}[TEST]${NC} $1"; }++# Check if test has exceeded maximum duration+check_timeout() {+ if [[ -z "${TEST_START_TIME}" ]]; then+ return 0+ fi+ local now+ now=$(date +%s)+ local elapsed=$((now - TEST_START_TIME))+ if [[ ${elapsed} -ge ${MAX_TEST_DURATION} ]]; then+ log_error "Test exceeded maximum duration of ${MAX_TEST_DURATION} seconds"+ exit 1+ fi+}++# Real ActivityPub accounts (these should resolve successfully)+REAL_ACCOUNTS=(+ # Original 5+ "https://social.gabekangas.com/users/gabek"+ "https://social.owncast.online/users/owncast"+ "https://sfba.social/users/dnalounge"+ "https://mastodon.social/users/docpop"+ "https://mastodon.social/users/Gargron"+ # Additional 20 real accounts from various instances+ "https://mastodon.social/users/Mastodon"+ "https://mastodon.social/users/joinmastodon"+ "https://mastodon.social/users/thunderbird"+ "https://mastodon.social/users/mozilla"+ "https://mastodon.social/users/pixelfed"+ "https://fosstodon.org/users/fosstodon"+ "https://fosstodon.org/users/kde"+ "https://fosstodon.org/users/gnome"+ "https://hachyderm.io/users/hachyderm"+ "https://hachyderm.io/users/nova"+ "https://infosec.exchange/users/jerry"+ "https://mas.to/users/Mer__edith"+ "https://mastodon.online/users/mastodonusercount"+ "https://mstdn.social/users/stux"+ "https://techhub.social/users/techhub"+ "https://universeodon.com/users/Universeodon"+ "https://mastodon.world/users/mastodonworld"+ "https://social.vivaldi.net/users/user_vivaldi"+ "https://c.im/users/publicvoit"+ "https://mastodon.social/users/w3c"+)++# Fake follower IRIs (these will fail validation)+FAKE_ACCOUNTS=(+ # Original 5+ "https://fake-server-1.invalid/users/fakeuser1"+ "https://fake-server-2.invalid/users/fakeuser2"+ "https://fake-server-3.invalid/users/fakeuser3"+ "https://nonexistent-instance.fake/users/nobody"+ "https://totally-fake.example/actors/testuser"+ # Additional 20 fake accounts+ "https://fake-server-4.invalid/users/fakeuser4"+ "https://fake-server-5.invalid/users/fakeuser5"+ "https://fake-server-6.invalid/users/fakeuser6"+ "https://fake-server-7.invalid/users/fakeuser7"+ "https://fake-server-8.invalid/users/fakeuser8"+ "https://fake-server-9.invalid/users/fakeuser9"+ "https://fake-server-10.invalid/users/fakeuser10"+ "https://nonexistent-1.fake/users/ghost1"+ "https://nonexistent-2.fake/users/ghost2"+ "https://nonexistent-3.fake/users/ghost3"+ "https://nonexistent-4.fake/users/ghost4"+ "https://nonexistent-5.fake/users/ghost5"+ "https://imaginary-instance.test/actors/phantom1"+ "https://imaginary-instance.test/actors/phantom2"+ "https://imaginary-instance.test/actors/phantom3"+ "https://madeup-server.invalid/users/invented1"+ "https://madeup-server.invalid/users/invented2"+ "https://madeup-server.invalid/users/invented3"+ "https://fictional-fedi.fake/users/notreal1"+ "https://fictional-fedi.fake/users/notreal2"+)++cleanup() {+ log_info "Cleaning up..."++ if [[ -n "${OWNCAST_PID}" ]] && kill -0 "${OWNCAST_PID}" 2>/dev/null; then+ kill "${OWNCAST_PID}" 2>/dev/null || true+ wait "${OWNCAST_PID}" 2>/dev/null || true+ fi++ if [[ -n "${TEMP_DIR}" ]] && [[ -d "${TEMP_DIR}" ]]; then+ rm -rf "${TEMP_DIR}"+ fi++ log_info "Cleanup complete."+}++trap cleanup EXIT++setup_temp_dir() {+ TEMP_DIR=$(mktemp -d)+ OWNCAST_DB="${TEMP_DIR}/owncast.db"+ log_info "Temp directory: ${TEMP_DIR}"+ log_info "Database: ${OWNCAST_DB}"+}++build_owncast() {+ log_info "Building Owncast..."++ pushd "${REPO_ROOT}" > /dev/null+ CGO_ENABLED=1 go build -o owncast main.go+ popd > /dev/null++ log_info "Owncast built"+}++start_owncast() {+ log_info "Starting Owncast with ${VALIDATION_INTERVAL}s validation interval..."++ # Start Owncast with test configuration+ OWNCAST_ALLOW_INTERNAL_FEDERATION=true \+ OWNCAST_INSECURE_SKIP_VERIFY=true \+ "${REPO_ROOT}/owncast" \+ -database "${OWNCAST_DB}" \+ -followervalidationinterval "${VALIDATION_INTERVAL}" \+ -enableVerboseLogging &+ OWNCAST_PID=$!++ log_info "Owncast started with PID ${OWNCAST_PID}"++ # Wait for Owncast to be ready+ local max_attempts=30+ local attempt=0++ while [[ ${attempt} -lt ${max_attempts} ]]; do+ if curl -s "http://localhost:${OWNCAST_PORT}/api/status" > /dev/null 2>&1; then+ log_info "Owncast is ready"+ return 0+ fi+ attempt=$((attempt + 1))+ sleep 1+ done++ log_error "Owncast did not become ready"+ return 1+}++stop_owncast() {+ log_info "Stopping Owncast..."+ if [[ -n "${OWNCAST_PID}" ]] && kill -0 "${OWNCAST_PID}" 2>/dev/null; then+ kill "${OWNCAST_PID}" 2>/dev/null || true+ wait "${OWNCAST_PID}" 2>/dev/null || true+ OWNCAST_PID=""+ fi+ # Give the database time to fully close+ sleep 2+ log_info "Owncast stopped"+}++configure_owncast() {+ log_info "Configuring Owncast federation..."++ local base_url="http://localhost:${OWNCAST_PORT}"+ local auth+ auth=$(echo -n "${ADMIN_USER}:${ADMIN_PASS}" | base64)++ # Set server URL+ curl -s -X POST "${base_url}/api/admin/config/serverurl" \+ -H "Authorization: Basic ${auth}" \+ -H "Content-Type: application/json" \+ -d '{"value": "http://localhost:8080"}' > /dev/null++ # Set federation username+ curl -s -X POST "${base_url}/api/admin/config/federation/username" \+ -H "Authorization: Basic ${auth}" \+ -H "Content-Type: application/json" \+ -d "{\"value\": \"${FEDERATION_USERNAME}\"}" > /dev/null++ # Enable federation+ curl -s -X POST "${base_url}/api/admin/config/federation/enable" \+ -H "Authorization: Basic ${auth}" \+ -H "Content-Type: application/json" \+ -d '{"value": true}' > /dev/null++ # Disable private mode (auto-accept follows)+ curl -s -X POST "${base_url}/api/admin/config/federation/private" \+ -H "Authorization: Basic ${auth}" \+ -H "Content-Type: application/json" \+ -d '{"value": false}' > /dev/null++ log_info "Owncast federation configured"+}++# Get the domain from an actor IRI+get_domain_from_iri() {+ local iri="$1"+ echo "${iri}" | sed -E 's|https?://([^/]+)/.*|\1|'+}++# Get the username from an actor IRI+get_username_from_iri() {+ local iri="$1"+ echo "${iri}" | sed -E 's|.*/([^/]+)$|\1|'+}++insert_followers() {+ log_info "Inserting test followers into database..."++ local now+ now=$(date -u +"%Y-%m-%d %H:%M:%S")++ # 8 days ago - past the 7-day threshold for removal+ local eight_days_ago+ eight_days_ago=$(date -u -d "8 days ago" +"%Y-%m-%d %H:%M:%S" 2>/dev/null || date -u -v-8d +"%Y-%m-%d %H:%M:%S")++ # Insert real followers (these should validate successfully and remain)+ log_info "Inserting ${#REAL_ACCOUNTS[@]} real followers..."+ for iri in "${REAL_ACCOUNTS[@]}"; do+ local domain+ domain=$(get_domain_from_iri "${iri}")+ local username+ username=$(get_username_from_iri "${iri}")+ local inbox="https://${domain}/inbox"+ local shared_inbox="https://${domain}/inbox"+ local request_iri="https://localhost:8080/federation/follow-request-${username}"++ sqlite3 "${OWNCAST_DB}" <<EOF+INSERT INTO ap_followers (iri, inbox, shared_inbox, name, username, image, request, request_object, created_at, approved_at)+VALUES (+ '${iri}',+ '${inbox}',+ '${shared_inbox}',+ '${username}',+ '${username}@${domain}',+ '',+ '${request_iri}',+ '{}',+ '${now}',+ '${now}'+);+EOF+ log_info " Inserted real follower: ${iri}"+ done++ # Insert fake followers with first_validation_failure_at set to 8 days ago+ # These should be removed after validation fails+ log_info "Inserting ${#FAKE_ACCOUNTS[@]} fake followers (with 8-day-old failure timestamp)..."+ for iri in "${FAKE_ACCOUNTS[@]}"; do+ local domain+ domain=$(get_domain_from_iri "${iri}")+ local username+ username=$(get_username_from_iri "${iri}")+ local inbox="https://${domain}/inbox"+ local request_iri="https://localhost:8080/federation/follow-request-${username}"++ sqlite3 "${OWNCAST_DB}" <<EOF+INSERT INTO ap_followers (iri, inbox, name, username, image, request, request_object, created_at, approved_at, first_validation_failure_at)+VALUES (+ '${iri}',+ '${inbox}',+ '${username}',+ '${username}@${domain}',+ '',+ '${request_iri}',+ '{}',+ '${now}',+ '${now}',+ '${eight_days_ago}'+);+EOF+ log_info " Inserted fake follower: ${iri}"+ done++ log_info "Inserted ${#REAL_ACCOUNTS[@]} real and ${#FAKE_ACCOUNTS[@]} fake followers"+}++# Get all follower IRIs via the admin API+# Returns a newline-separated list of follower IRIs+get_followers_via_api() {+ local base_url="http://localhost:${OWNCAST_PORT}"+ local auth+ auth=$(echo -n "${ADMIN_USER}:${ADMIN_PASS}" | base64)++ # Fetch all followers (using high limit to get all at once)+ local response+ response=$(curl -s "${base_url}/api/admin/followers?limit=1000" \+ -H "Authorization: Basic ${auth}" 2>/dev/null || echo '{"results":[]}')++ # Extract IRIs from the JSON response+ echo "${response}" | jq -r '.results[]?.link // empty' 2>/dev/null || echo ""+}++get_follower_count() {+ local followers+ followers=$(get_followers_via_api)+ if [[ -z "${followers}" ]]; then+ echo "0"+ else+ echo "${followers}" | wc -l | tr -d ' '+ fi+}++get_real_follower_count() {+ # Count followers that match real account patterns+ local followers+ followers=$(get_followers_via_api)+ local count=0+ for iri in "${REAL_ACCOUNTS[@]}"; do+ if echo "${followers}" | grep -qF "${iri}"; then+ count=$((count + 1))+ fi+ done+ echo "${count}"+}++get_fake_follower_count() {+ # Count followers that match fake account patterns+ local followers+ followers=$(get_followers_via_api)+ local count=0+ for iri in "${FAKE_ACCOUNTS[@]}"; do+ if echo "${followers}" | grep -qF "${iri}"; then+ count=$((count + 1))+ fi+ done+ echo "${count}"+}++wait_for_validation() {+ log_info "Waiting for follower validation job to run..."++ local _initial_fake_count+ _initial_fake_count=$(get_fake_follower_count)++ # Calculate wait time based on number of followers+ # The job validates 5 followers per run (FollowersPerRun)+ # With 2-second delay between followers and VALIDATION_INTERVAL between runs+ local total_followers=$((${#REAL_ACCOUNTS[@]} + ${#FAKE_ACCOUNTS[@]}))+ local cycles_needed=$(( (total_followers + 4) / 5 )) # Round up+ local wait_time=$(( (VALIDATION_INTERVAL + 12) * cycles_needed + 30 )) # Extra buffer for network delays++ log_info "Total followers: ${total_followers}, estimated cycles needed: ${cycles_needed}"+ log_info "Waiting up to ${wait_time} seconds for validation cycles..."++ local elapsed=0+ local check_interval=10++ while [[ ${elapsed} -lt ${wait_time} ]]; do+ # Check if we've exceeded the maximum test duration+ check_timeout++ sleep ${check_interval}+ elapsed=$((elapsed + check_interval))++ local current_fake_count+ current_fake_count=$(get_fake_follower_count)+ local current_real_count+ current_real_count=$(get_real_follower_count)++ log_info " ${elapsed}s: Real followers: ${current_real_count}/${#REAL_ACCOUNTS[@]}, Fake followers: ${current_fake_count}/${#FAKE_ACCOUNTS[@]}"++ # If all fake followers are gone and all real remain, we can stop early+ if [[ ${current_fake_count} -eq 0 ]] && [[ ${current_real_count} -eq ${#REAL_ACCOUNTS[@]} ]]; then+ log_info "Validation complete early - all fake followers removed"+ return 0+ fi+ done++ return 0+}++verify_results() {+ log_info "Verifying validation results..."++ local total_count+ total_count=$(get_follower_count)+ local real_count+ real_count=$(get_real_follower_count)+ local fake_count+ fake_count=$(get_fake_follower_count)++ log_test "Total followers: ${total_count}"+ log_test "Real followers remaining: ${real_count}/${#REAL_ACCOUNTS[@]}"+ log_test "Fake followers remaining: ${fake_count}/${#FAKE_ACCOUNTS[@]}"++ # List remaining followers for debugging+ log_info "Remaining followers:"+ get_followers_via_api | while read -r iri; do+ if [[ -n "${iri}" ]]; then+ log_info " - ${iri}"+ fi+ done++ echo ""+ echo "========================================"+ echo "Follower Validation Test Results"+ echo "========================================"+ echo "Expected real followers: ${#REAL_ACCOUNTS[@]}"+ echo "Actual real followers: ${real_count}"+ echo "Expected fake followers: 0"+ echo "Actual fake followers: ${fake_count}"+ echo "========================================"++ local passed=true++ # Check that all fake followers were removed+ if [[ ${fake_count} -ne 0 ]]; then+ log_error "FAIL: ${fake_count} fake followers should have been removed"+ passed=false+ else+ log_test "PASS: All fake followers were removed"+ fi++ # Check that all real followers remain+ if [[ ${real_count} -ne ${#REAL_ACCOUNTS[@]} ]]; then+ log_error "FAIL: Only ${real_count}/${#REAL_ACCOUNTS[@]} real followers remain"+ passed=false+ else+ log_test "PASS: All real followers remain"+ fi++ echo ""+ if [[ "${passed}" == "true" ]]; then+ echo -e "${GREEN}TEST PASSED${NC}"+ return 0+ else+ echo -e "${RED}TEST FAILED${NC}"+ return 1+ fi+}++main() {+ # Record test start time for timeout checking+ TEST_START_TIME=$(date +%s)++ echo ""+ echo "========================================"+ echo "Follower Validation Test"+ echo "========================================"+ echo ""++ # Setup+ setup_temp_dir+ build_owncast+ start_owncast+ configure_owncast++ # Stop Owncast before inserting test data to avoid database contention+ # The database schema is now created, so we can safely insert data+ stop_owncast++ # Insert test data while Owncast is stopped+ insert_followers++ # Restart Owncast to run the validation job+ start_owncast++ # Verify initial state+ log_info "Initial state:"+ log_info " Total followers: $(get_follower_count)"+ log_info " Real followers: $(get_real_follower_count)"+ log_info " Fake followers: $(get_fake_follower_count)"++ # Wait for validation+ wait_for_validation++ # Verify results+ if verify_results; then+ exit 0+ else+ exit 1+ fi+}++main "$@"