Take control over your live stream video by running it yourself. Streaming + chat out of the box.
HTML activitypubbroadcastingchatdecentralizedfederationfediversegolanghacktoberfesthlslivelivestreamowncastrtmpself-hostedstreaming-videovideo
fix(datastore): retain database backup history
Gabe Kangas3c2fd7b088ca
diff --git a/services/datastore/backup_test.go b/services/datastore/backup_test.gonew file mode 100644index 0000000000..910962190d--- /dev/null+++ b/services/datastore/backup_test.go@@ -0,0 +1,107 @@+package datastore++import (+ "database/sql"+ "fmt"+ "os"+ "path/filepath"+ "testing"+ "time"++ _ "github.com/mattn/go-sqlite3"+ "github.com/owncast/owncast/utils"+)++func TestBackupDatabaseAtKeepsCurrentAndHistory(t *testing.T) {+ db, err := sql.Open("sqlite3", ":memory:")+ if err != nil {+ t.Fatal(err)+ }+ db.SetMaxOpenConns(1)+ defer db.Close()+ if _, err := db.Exec("CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('old');"); err != nil {+ t.Fatal(err)+ }++ backupDirectory := t.TempDir()+ firstBackupTime := time.Date(2026, 8, 17, 18, 0, 0, 0, time.UTC)+ backupDatabaseAt(db, backupDirectory, firstBackupTime)++ if _, err := db.Exec("UPDATE values_table SET value = 'new'"); err != nil {+ t.Fatal(err)+ }+ secondBackupTime := firstBackupTime.Add(time.Hour)+ backupDatabaseAt(db, backupDirectory, secondBackupTime)++ historyFile := filepath.Join(backupDirectory, "owncastdb-20260817T190000Z.bak")+ if _, err := os.Stat(historyFile); err != nil {+ t.Fatalf("history backup missing: %v", err)+ }++ historyDatabase := filepath.Join(t.TempDir(), "history.db")+ if err := utils.Restore(historyFile, historyDatabase); err != nil {+ t.Fatal(err)+ }+ assertBackupValue(t, historyDatabase, "old")++ currentDatabase := filepath.Join(t.TempDir(), "current.db")+ if err := utils.Restore(filepath.Join(backupDirectory, databaseBackupFileName), currentDatabase); err != nil {+ t.Fatal(err)+ }+ assertBackupValue(t, currentDatabase, "new")+}++func TestBackupDatabaseAtPrunesOldHistory(t *testing.T) {+ db, err := sql.Open("sqlite3", ":memory:")+ if err != nil {+ t.Fatal(err)+ }+ db.SetMaxOpenConns(1)+ defer db.Close()+ if _, err := db.Exec("CREATE TABLE values_table (value TEXT)"); err != nil {+ t.Fatal(err)+ }++ backupDirectory := t.TempDir()+ firstBackupTime := time.Date(2026, 8, 17, 0, 0, 0, 0, time.UTC)+ for i := range 25 {+ if _, err := db.Exec("DELETE FROM values_table"); err != nil {+ t.Fatal(err)+ }+ if _, err := db.Exec("INSERT INTO values_table VALUES (?)", fmt.Sprintf("backup-%d", i)); err != nil {+ t.Fatal(err)+ }+ backupDatabaseAt(db, backupDirectory, firstBackupTime.Add(time.Duration(i)*time.Hour))+ }++ entries, err := os.ReadDir(backupDirectory)+ if err != nil {+ t.Fatal(err)+ }+ if len(entries) != maxDatabaseBackups {+ t.Fatalf("backup file count = %d, want %d", len(entries), maxDatabaseBackups)+ }+ if _, err := os.Stat(filepath.Join(backupDirectory, "owncastdb-20260817T010000Z.bak")); !os.IsNotExist(err) {+ t.Fatalf("oldest history backup still exists, stat error = %v", err)+ }+ if _, err := os.Stat(filepath.Join(backupDirectory, "owncastdb-20260817T020000Z.bak")); err != nil {+ t.Fatalf("newest retained history backup missing: %v", err)+ }+}++func assertBackupValue(t *testing.T, databaseFile, want string) {+ t.Helper()+ db, err := sql.Open("sqlite3", databaseFile)+ if err != nil {+ t.Fatal(err)+ }+ defer db.Close()++ var got string+ if err := db.QueryRow("SELECT value FROM values_table").Scan(&got); err != nil {+ t.Fatal(err)+ }+ if got != want {+ t.Fatalf("backup value = %q, want %q", got, want)+ }+}diff --git a/services/datastore/data.go b/services/datastore/data.goindex 0c90cfd08d..ec6c34e15c 100644--- a/services/datastore/data.go+++ b/services/datastore/data.go@@ -11,6 +11,8 @@ import ("fmt""os""path/filepath"+ "sort"+ "strings""time"log "github.com/sirupsen/logrus"@@ -22,7 +24,7 @@ import (// SetupPersistence opens the datastore file (or an in-memory database// when file == ":memory:"), runs migrations, starts the periodic backup// goroutine, and returns the constructed *Datastore. backupDirectory is-// the directory the hourly backup goroutine writes owncastdb.bak into.+// the directory where the current and retained hourly backups are written.// The returned value is the sole handle to the database — main.go// threads it through every consumer.func SetupPersistence(file, backupDirectory string) (*Datastore, error) {@@ -74,13 +76,64 @@ func SetupPersistence(file, backupDirectory string) (*Datastore, error) {dataStore := &Datastore{}dataStore.Setup(db)- dbBackupTicker := time.NewTicker(1 * time.Hour)+ dbBackupTicker := time.NewTicker(time.Hour)go func() {- backupFile := filepath.Join(backupDirectory, "owncastdb.bak")- for range dbBackupTicker.C {- utils.Backup(db, backupFile)+ for backupTime := range dbBackupTicker.C {+ backupDatabaseAt(db, backupDirectory, backupTime)}}()return dataStore, nil}++const (+ databaseBackupFileName = "owncastdb.bak"+ databaseBackupHistoryPrefix = "owncastdb-"+ databaseBackupHistorySuffix = ".bak"+ maxDatabaseBackups = 24+)++func backupDatabaseAt(db *sql.DB, backupDirectory string, backupTime time.Time) {+ if err := os.MkdirAll(backupDirectory, 0o700); err != nil {+ log.Errorln("unable to create backup directory", backupDirectory, err)+ return+ }++ backupFile := filepath.Join(backupDirectory, databaseBackupFileName)+ if utils.DoesFileExists(backupFile) {+ historyFile := filepath.Join(+ backupDirectory,+ fmt.Sprintf("%s%s%s", databaseBackupHistoryPrefix, backupTime.UTC().Format("20060102T150405Z"), databaseBackupHistorySuffix),+ )+ if err := utils.Copy(backupFile, historyFile); err != nil {+ log.Errorln("unable to preserve previous database backup", err)+ return+ }+ }++ utils.Backup(db, backupFile)++ entries, err := os.ReadDir(backupDirectory)+ if err != nil {+ log.Errorln("unable to read database backup directory", err)+ return+ }++ var historyFiles []string+ for _, entry := range entries {+ if entry.IsDir() || !strings.HasPrefix(entry.Name(), databaseBackupHistoryPrefix) || !strings.HasSuffix(entry.Name(), databaseBackupHistorySuffix) {+ continue+ }+ historyFiles = append(historyFiles, entry.Name())+ }+ sort.Strings(historyFiles)++ for len(historyFiles) > maxDatabaseBackups-1 {+ oldest := filepath.Join(backupDirectory, historyFiles[0])+ if err := os.Remove(oldest); err != nil {+ log.Errorln("unable to remove old database backup", oldest, err)+ return+ }+ historyFiles = historyFiles[1:]+ }+}diff --git a/utils/backup.go b/utils/backup.goindex fe6de1c47e..a4f2747dd1 100644--- a/utils/backup.go+++ b/utils/backup.go@@ -28,6 +28,8 @@ func Restore(backupFile string, databaseFile string) error {if err != nil {return fmt.Errorf("unable to read backup file %s", err)}+ // Older Owncast backups can contain trailing bytes from an untruncated overwrite.+ gz.Multistream(false)defer gz.Close()var b bytes.Buffer@@ -80,7 +82,7 @@ func Backup(db *sql.DB, backupFile string) {_ = out.Flush()// Create a new backup file- f, err := os.OpenFile(backupFile, os.O_WRONLY|os.O_CREATE, 0o600) // nolint+ f, err := os.OpenFile(backupFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) // nolintif err != nil {handleError(err)returndiff --git a/utils/backup_test.go b/utils/backup_test.gonew file mode 100644index 0000000000..344f941b4b--- /dev/null+++ b/utils/backup_test.go@@ -0,0 +1,115 @@+package utils++import (+ "bytes"+ "compress/gzip"+ "database/sql"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "testing"++ _ "github.com/mattn/go-sqlite3"+)++func TestRestoreAcceptsTrailingDataFromLegacyBackup(t *testing.T) {+ db, err := sql.Open("sqlite3", ":memory:")+ if err != nil {+ t.Fatal(err)+ }+ db.SetMaxOpenConns(1)+ defer db.Close()+ if _, err := db.Exec("CREATE TABLE restored (value TEXT); INSERT INTO restored VALUES ('ok');"); err != nil {+ t.Fatal(err)+ }++ backupFile := filepath.Join(t.TempDir(), "owncastdb.bak")+ Backup(db, backupFile)++ compressed, err := os.ReadFile(backupFile)+ if err != nil {+ t.Fatal(err)+ }+ compressed = append(compressed, []byte("trailing garbage from an older backup")...)+ if err := os.WriteFile(backupFile, compressed, 0o600); err != nil {+ t.Fatal(err)+ }++ databaseFile := filepath.Join(t.TempDir(), "owncast.db")+ if err := Restore(backupFile, databaseFile); err != nil {+ t.Fatalf("Restore() error = %v", err)+ }++ restoredDB, err := sql.Open("sqlite3", databaseFile)+ if err != nil {+ t.Fatal(err)+ }+ defer restoredDB.Close()++ var value string+ if err := restoredDB.QueryRow("SELECT value FROM restored").Scan(&value); err != nil {+ t.Fatal(err)+ }+ if value != "ok" {+ t.Fatalf("restored value = %q, want %q", value, "ok")+ }+}++func TestBackupTruncatesExistingFile(t *testing.T) {+ db, err := sql.Open("sqlite3", ":memory:")+ if err != nil {+ t.Fatal(err)+ }+ db.SetMaxOpenConns(1)+ defer db.Close()+ if _, err := db.Exec("CREATE TABLE restored (value TEXT)"); err != nil {+ t.Fatal(err)+ }+ for i := range 512 {+ if _, err := db.Exec("INSERT INTO restored VALUES (?)", fmt.Sprintf("row-%08d-%08d", i, i*7919)); err != nil {+ t.Fatal(err)+ }+ }++ backupFile := filepath.Join(t.TempDir(), "owncastdb.bak")+ Backup(db, backupFile)+ if _, err := db.Exec("DELETE FROM restored"); err != nil {+ t.Fatal(err)+ }+ Backup(db, backupFile)+ secondBackup, err := os.ReadFile(backupFile)+ if err != nil {+ t.Fatal(err)+ }++ gzipReader, err := gzip.NewReader(bytes.NewReader(secondBackup))+ if err != nil {+ t.Fatal(err)+ }+ if _, err := io.Copy(io.Discard, gzipReader); err != nil {+ t.Fatalf("new backup contains trailing data: %v", err)+ }+ if err := gzipReader.Close(); err != nil {+ t.Fatal(err)+ }++ databaseFile := filepath.Join(t.TempDir(), "restored.db")+ if err := Restore(backupFile, databaseFile); err != nil {+ t.Fatalf("round-trip Restore() error = %v", err)+ }++ restoredDB, err := sql.Open("sqlite3", databaseFile)+ if err != nil {+ t.Fatal(err)+ }+ defer restoredDB.Close()++ var count int+ if err := restoredDB.QueryRow("SELECT COUNT(*) FROM restored").Scan(&count); err != nil {+ t.Fatal(err)+ }+ if count != 0 {+ t.Fatalf("round-trip row count = %d, want 0", count)+ }+}