Skip to content

Take control over your live stream video by running it yourself. Streaming + chat out of the box.

fix(datastore): retain database backup history

Gabe Kangas3c2fd7b088ca
diff --git a/services/datastore/backup_test.go b/services/datastore/backup_test.go
new file mode 100644
index 0000000000..910962190d
--- /dev/null
+++ b/services/datastore/backup_test.go
@@ -0,0 +1,107 @@
+package datastore
+
+import (
+ "database/sql"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ _ "github.com/mattn/go-sqlite3"
+ "github.com/owncast/owncast/utils"
+)
+
+func TestBackupDatabaseAtKeepsCurrentAndHistory(t *testing.T) {
+ db, err := sql.Open("sqlite3", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ db.SetMaxOpenConns(1)
+ defer db.Close()
+ if _, err := db.Exec("CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('old');"); err != nil {
+ t.Fatal(err)
+ }
+
+ backupDirectory := t.TempDir()
+ firstBackupTime := time.Date(2026, 8, 17, 18, 0, 0, 0, time.UTC)
+ backupDatabaseAt(db, backupDirectory, firstBackupTime)
+
+ if _, err := db.Exec("UPDATE values_table SET value = 'new'"); err != nil {
+ t.Fatal(err)
+ }
+ secondBackupTime := firstBackupTime.Add(time.Hour)
+ backupDatabaseAt(db, backupDirectory, secondBackupTime)
+
+ historyFile := filepath.Join(backupDirectory, "owncastdb-20260817T190000Z.bak")
+ if _, err := os.Stat(historyFile); err != nil {
+ t.Fatalf("history backup missing: %v", err)
+ }
+
+ historyDatabase := filepath.Join(t.TempDir(), "history.db")
+ if err := utils.Restore(historyFile, historyDatabase); err != nil {
+ t.Fatal(err)
+ }
+ assertBackupValue(t, historyDatabase, "old")
+
+ currentDatabase := filepath.Join(t.TempDir(), "current.db")
+ if err := utils.Restore(filepath.Join(backupDirectory, databaseBackupFileName), currentDatabase); err != nil {
+ t.Fatal(err)
+ }
+ assertBackupValue(t, currentDatabase, "new")
+}
+
+func TestBackupDatabaseAtPrunesOldHistory(t *testing.T) {
+ db, err := sql.Open("sqlite3", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ db.SetMaxOpenConns(1)
+ defer db.Close()
+ if _, err := db.Exec("CREATE TABLE values_table (value TEXT)"); err != nil {
+ t.Fatal(err)
+ }
+
+ backupDirectory := t.TempDir()
+ firstBackupTime := time.Date(2026, 8, 17, 0, 0, 0, 0, time.UTC)
+ for i := range 25 {
+ if _, err := db.Exec("DELETE FROM values_table"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec("INSERT INTO values_table VALUES (?)", fmt.Sprintf("backup-%d", i)); err != nil {
+ t.Fatal(err)
+ }
+ backupDatabaseAt(db, backupDirectory, firstBackupTime.Add(time.Duration(i)*time.Hour))
+ }
+
+ entries, err := os.ReadDir(backupDirectory)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(entries) != maxDatabaseBackups {
+ t.Fatalf("backup file count = %d, want %d", len(entries), maxDatabaseBackups)
+ }
+ if _, err := os.Stat(filepath.Join(backupDirectory, "owncastdb-20260817T010000Z.bak")); !os.IsNotExist(err) {
+ t.Fatalf("oldest history backup still exists, stat error = %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(backupDirectory, "owncastdb-20260817T020000Z.bak")); err != nil {
+ t.Fatalf("newest retained history backup missing: %v", err)
+ }
+}
+
+func assertBackupValue(t *testing.T, databaseFile, want string) {
+ t.Helper()
+ db, err := sql.Open("sqlite3", databaseFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+
+ var got string
+ if err := db.QueryRow("SELECT value FROM values_table").Scan(&got); err != nil {
+ t.Fatal(err)
+ }
+ if got != want {
+ t.Fatalf("backup value = %q, want %q", got, want)
+ }
+}
diff --git a/services/datastore/data.go b/services/datastore/data.go
index 0c90cfd08d..ec6c34e15c 100644
--- a/services/datastore/data.go
+++ b/services/datastore/data.go
@@ -11,6 +11,8 @@ import (
"fmt"
"os"
"path/filepath"
+ "sort"
+ "strings"
"time"
log "github.com/sirupsen/logrus"
@@ -22,7 +24,7 @@ import (
// SetupPersistence opens the datastore file (or an in-memory database
// when file == ":memory:"), runs migrations, starts the periodic backup
// goroutine, and returns the constructed *Datastore. backupDirectory is
-// the directory the hourly backup goroutine writes owncastdb.bak into.
+// the directory where the current and retained hourly backups are written.
// The returned value is the sole handle to the database — main.go
// threads it through every consumer.
func SetupPersistence(file, backupDirectory string) (*Datastore, error) {
@@ -74,13 +76,64 @@ func SetupPersistence(file, backupDirectory string) (*Datastore, error) {
dataStore := &Datastore{}
dataStore.Setup(db)
- dbBackupTicker := time.NewTicker(1 * time.Hour)
+ dbBackupTicker := time.NewTicker(time.Hour)
go func() {
- backupFile := filepath.Join(backupDirectory, "owncastdb.bak")
- for range dbBackupTicker.C {
- utils.Backup(db, backupFile)
+ for backupTime := range dbBackupTicker.C {
+ backupDatabaseAt(db, backupDirectory, backupTime)
}
}()
return dataStore, nil
}
+
+const (
+ databaseBackupFileName = "owncastdb.bak"
+ databaseBackupHistoryPrefix = "owncastdb-"
+ databaseBackupHistorySuffix = ".bak"
+ maxDatabaseBackups = 24
+)
+
+func backupDatabaseAt(db *sql.DB, backupDirectory string, backupTime time.Time) {
+ if err := os.MkdirAll(backupDirectory, 0o700); err != nil {
+ log.Errorln("unable to create backup directory", backupDirectory, err)
+ return
+ }
+
+ backupFile := filepath.Join(backupDirectory, databaseBackupFileName)
+ if utils.DoesFileExists(backupFile) {
+ historyFile := filepath.Join(
+ backupDirectory,
+ fmt.Sprintf("%s%s%s", databaseBackupHistoryPrefix, backupTime.UTC().Format("20060102T150405Z"), databaseBackupHistorySuffix),
+ )
+ if err := utils.Copy(backupFile, historyFile); err != nil {
+ log.Errorln("unable to preserve previous database backup", err)
+ return
+ }
+ }
+
+ utils.Backup(db, backupFile)
+
+ entries, err := os.ReadDir(backupDirectory)
+ if err != nil {
+ log.Errorln("unable to read database backup directory", err)
+ return
+ }
+
+ var historyFiles []string
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasPrefix(entry.Name(), databaseBackupHistoryPrefix) || !strings.HasSuffix(entry.Name(), databaseBackupHistorySuffix) {
+ continue
+ }
+ historyFiles = append(historyFiles, entry.Name())
+ }
+ sort.Strings(historyFiles)
+
+ for len(historyFiles) > maxDatabaseBackups-1 {
+ oldest := filepath.Join(backupDirectory, historyFiles[0])
+ if err := os.Remove(oldest); err != nil {
+ log.Errorln("unable to remove old database backup", oldest, err)
+ return
+ }
+ historyFiles = historyFiles[1:]
+ }
+}
diff --git a/utils/backup.go b/utils/backup.go
index fe6de1c47e..a4f2747dd1 100644
--- a/utils/backup.go
+++ b/utils/backup.go
@@ -28,6 +28,8 @@ func Restore(backupFile string, databaseFile string) error {
if err != nil {
return fmt.Errorf("unable to read backup file %s", err)
}
+ // Older Owncast backups can contain trailing bytes from an untruncated overwrite.
+ gz.Multistream(false)
defer gz.Close()
var b bytes.Buffer
@@ -80,7 +82,7 @@ func Backup(db *sql.DB, backupFile string) {
_ = out.Flush()
// Create a new backup file
- f, err := os.OpenFile(backupFile, os.O_WRONLY|os.O_CREATE, 0o600) // nolint
+ f, err := os.OpenFile(backupFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) // nolint
if err != nil {
handleError(err)
return
diff --git a/utils/backup_test.go b/utils/backup_test.go
new file mode 100644
index 0000000000..344f941b4b
--- /dev/null
+++ b/utils/backup_test.go
@@ -0,0 +1,115 @@
+package utils
+
+import (
+ "bytes"
+ "compress/gzip"
+ "database/sql"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "testing"
+
+ _ "github.com/mattn/go-sqlite3"
+)
+
+func TestRestoreAcceptsTrailingDataFromLegacyBackup(t *testing.T) {
+ db, err := sql.Open("sqlite3", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ db.SetMaxOpenConns(1)
+ defer db.Close()
+ if _, err := db.Exec("CREATE TABLE restored (value TEXT); INSERT INTO restored VALUES ('ok');"); err != nil {
+ t.Fatal(err)
+ }
+
+ backupFile := filepath.Join(t.TempDir(), "owncastdb.bak")
+ Backup(db, backupFile)
+
+ compressed, err := os.ReadFile(backupFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ compressed = append(compressed, []byte("trailing garbage from an older backup")...)
+ if err := os.WriteFile(backupFile, compressed, 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ databaseFile := filepath.Join(t.TempDir(), "owncast.db")
+ if err := Restore(backupFile, databaseFile); err != nil {
+ t.Fatalf("Restore() error = %v", err)
+ }
+
+ restoredDB, err := sql.Open("sqlite3", databaseFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer restoredDB.Close()
+
+ var value string
+ if err := restoredDB.QueryRow("SELECT value FROM restored").Scan(&value); err != nil {
+ t.Fatal(err)
+ }
+ if value != "ok" {
+ t.Fatalf("restored value = %q, want %q", value, "ok")
+ }
+}
+
+func TestBackupTruncatesExistingFile(t *testing.T) {
+ db, err := sql.Open("sqlite3", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ db.SetMaxOpenConns(1)
+ defer db.Close()
+ if _, err := db.Exec("CREATE TABLE restored (value TEXT)"); err != nil {
+ t.Fatal(err)
+ }
+ for i := range 512 {
+ if _, err := db.Exec("INSERT INTO restored VALUES (?)", fmt.Sprintf("row-%08d-%08d", i, i*7919)); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ backupFile := filepath.Join(t.TempDir(), "owncastdb.bak")
+ Backup(db, backupFile)
+ if _, err := db.Exec("DELETE FROM restored"); err != nil {
+ t.Fatal(err)
+ }
+ Backup(db, backupFile)
+ secondBackup, err := os.ReadFile(backupFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ gzipReader, err := gzip.NewReader(bytes.NewReader(secondBackup))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := io.Copy(io.Discard, gzipReader); err != nil {
+ t.Fatalf("new backup contains trailing data: %v", err)
+ }
+ if err := gzipReader.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ databaseFile := filepath.Join(t.TempDir(), "restored.db")
+ if err := Restore(backupFile, databaseFile); err != nil {
+ t.Fatalf("round-trip Restore() error = %v", err)
+ }
+
+ restoredDB, err := sql.Open("sqlite3", databaseFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer restoredDB.Close()
+
+ var count int
+ if err := restoredDB.QueryRow("SELECT COUNT(*) FROM restored").Scan(&count); err != nil {
+ t.Fatal(err)
+ }
+ if count != 0 {
+ t.Fatalf("round-trip row count = %d, want 0", count)
+ }
+}