PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FEATURE (tests): Run tests against test dev DB each time to not flood DB where development is performed
Rostislav Dugina044b3bc2caa
diff --git a/.env.example b/.env.exampleindex 80c3a939..576ac9f9 100644--- a/.env.example+++ b/.env.example@@ -10,9 +10,13 @@ DEV_DB_PASSWORD=Q1234567DATABASE_DSN=host=localhost user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disableDATABASE_URL=postgres://postgres:Q1234567@localhost:5437/databasus?sslmode=disable+# test DB (used by backend when IsTesting=true so tests don't pollute the dev DB)+TEST_DATABASE_DSN=host=localhost user=postgres password=Q1234567 dbname=databasus port=5438 sslmode=disable+# migrationsGOOSE_DRIVER=postgresGOOSE_DBSTRING=postgres://postgres:Q1234567@localhost:5437/databasus?sslmode=disable+GOOSE_TEST_DBSTRING=postgres://postgres:Q1234567@localhost:5438/databasus?sslmode=disable# valkeyVALKEY_HOST=127.0.0.1diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.ymlindex 2041cb71..52baaf25 100644--- a/.github/workflows/ci-release.yml+++ b/.github/workflows/ci-release.yml@@ -266,6 +266,8 @@ jobs:DATABASE_DSN=host=172.17.0.1 user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disableDATABASE_URL=postgres://postgres:[email protected]:5437/databasus?sslmode=disableGOOSE_DBSTRING=postgres://postgres:[email protected]:5437/databasus?sslmode=disable+ TEST_DATABASE_DSN=host=172.17.0.1 user=postgres password=Q1234567 dbname=databasus port=5438 sslmode=disable+ GOOSE_TEST_DBSTRING=postgres://postgres:[email protected]:5438/databasus?sslmode=disableTEST_LOCALHOST=172.17.0.1TEST_DB_HOST=172.17.0.1VALKEY_HOST=172.17.0.1@@ -280,6 +282,9 @@ jobs:# Wait for main dev databasetimeout 60 bash -c 'until docker exec backend-db pg_isready -h localhost -p 5437 -U postgres; do sleep 2; done'+ # Wait for test database+ timeout 60 bash -c 'until docker exec backend-db-test pg_isready -h localhost -p 5438 -U postgres; do sleep 2; done'+# Wait for Valkey (cache)echo "Waiting for Valkey..."timeout 60 bash -c 'until docker exec backend-valkey valkey-cli ping 2>/dev/null | grep -q PONG; do sleep 2; done'@@ -371,7 +376,8 @@ jobs:run: |cd backendgo install github.com/pressly/goose/v3/cmd/[email protected]- goose -env ../.env -dir ./migrations up+ set -a && . ../.env && set +a+ goose -dir ./migrations postgres "$GOOSE_TEST_DBSTRING" up- name: Run Go testsrun: |@@ -387,6 +393,7 @@ jobs:# Clean up all data directories created by docker-composeecho "Cleaning up data directories..."rm -rf pgdata || true+ rm -rf pgdata-test || truerm -rf valkey-data || truerm -rf mysqldata || truerm -rf mariadbdata || truediff --git a/.gitignore b/.gitignoreindex 8628ddf5..d474735b 100644--- a/.gitignore+++ b/.gitignore@@ -2,6 +2,7 @@ ansible/databasus-data/.envpgdata/+pgdata-test/mysqldata/mariadbdata/valkey-data/diff --git a/backend/Makefile b/backend/Makefileindex b3762b48..661d21fa 100644--- a/backend/Makefile+++ b/backend/Makefile@@ -4,6 +4,8 @@ run:go run cmd/main.gotest:+ go run ./cmd/cleanup_test_db+ goose -dir ./migrations postgres "$(GOOSE_TEST_DBSTRING)" upgo test -p=1 -count=1 -failfast -timeout 15m ./internal/...lint:@@ -15,6 +17,9 @@ migration-create:migration-up:goose -dir ./migrations postgres "$(GOOSE_DBSTRING)" up+migration-up-test:+ goose -dir ./migrations postgres "$(GOOSE_TEST_DBSTRING)" up+migration-down:goose -dir ./migrations postgres "$(GOOSE_DBSTRING)" downdiff --git a/backend/cmd/cleanup_test_db/main.go b/backend/cmd/cleanup_test_db/main.gonew file mode 100644index 00000000..554e6af6--- /dev/null+++ b/backend/cmd/cleanup_test_db/main.go@@ -0,0 +1,92 @@+// Resets the test database before `make test` runs.+//+// Reads the test DSN through config (config.GetEnv() auto-swaps DatabaseDsn to+// TestDatabaseDsn when IsTesting is true). IsTesting is detected from os.Args+// containing the substring "test" — the binary path "cleanup_test_db" satisfies+// that. Renaming the binary or its directory will break detection.+package main++import (+ "fmt"+ "os"+ "strings"++ "gorm.io/driver/postgres"+ "gorm.io/gorm"+ gormLogger "gorm.io/gorm/logger"++ "databasus-backend/internal/config"+ "databasus-backend/internal/util/logger"+)++const systemDb = "postgres"++func main() {+ log := logger.GetLogger()++ env := config.GetEnv()+ if !env.IsTesting {+ log.Error("cleanup_test_db must run with IsTesting=true (binary name must contain 'test')")+ os.Exit(1)+ }++ targetDbName, systemDsn, err := rewriteDbName(env.DatabaseDsn, systemDb)+ if err != nil {+ log.Error("failed to parse TEST_DATABASE_DSN", "error", err)+ os.Exit(1)+ }++ log.Info("resetting test DB", "db", targetDbName)++ db, err := gorm.Open(postgres.Open(systemDsn), &gorm.Config{+ Logger: gormLogger.Default.LogMode(gormLogger.Silent),+ })+ if err != nil {+ log.Error("failed to connect to system postgres DB", "error", err)+ os.Exit(1)+ }++ quoted := quoteIdentifier(targetDbName)++ if err := db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s WITH (FORCE)", quoted)).Error; err != nil {+ log.Error("DROP DATABASE failed", "error", err)+ os.Exit(1)+ }++ if err := db.Exec(fmt.Sprintf("CREATE DATABASE %s", quoted)).Error; err != nil {+ log.Error("CREATE DATABASE failed", "error", err)+ os.Exit(1)+ }++ log.Info("test DB reset complete", "db", targetDbName)+}++func rewriteDbName(dsn, newDbName string) (origDbName, rewritten string, err error) {+ parts := strings.Fields(dsn)+ out := make([]string, 0, len(parts))++ for _, p := range parts {+ k, v, ok := strings.Cut(p, "=")+ if !ok {+ return "", "", fmt.Errorf("invalid DSN token: %q", p)+ }++ if k == "dbname" {+ origDbName = v+ out = append(out, "dbname="+newDbName)+ continue+ }++ out = append(out, p)+ }++ if origDbName == "" {+ return "", "", fmt.Errorf("DSN missing dbname: %q", dsn)+ }++ return origDbName, strings.Join(out, " "), nil+}++func quoteIdentifier(s string) string {+ return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`+}diff --git a/backend/internal/config/config.go b/backend/internal/config/config.goindex 7fe7035b..b946ae2d 100644--- a/backend/internal/config/config.go+++ b/backend/internal/config/config.go@@ -27,7 +27,8 @@ type EnvVariables struct {EnvMode env_utils.EnvMode `env:"ENV_MODE" required:"true"`// Internal database- DatabaseDsn string `env:"DATABASE_DSN" required:"true"`+ DatabaseDsn string `env:"DATABASE_DSN" required:"true"`+ TestDatabaseDsn string `env:"TEST_DATABASE_DSN"`// Internal ValkeyValkeyHost string `env:"VALKEY_HOST" required:"true"`ValkeyPort string `env:"VALKEY_PORT" required:"true"`@@ -199,6 +200,15 @@ func loadEnvVariables() {}}+ if env.IsTesting {+ if env.TestDatabaseDsn == "" {+ log.Error("TEST_DATABASE_DSN is empty")+ os.Exit(1)+ }++ env.DatabaseDsn = env.TestDatabaseDsn+ }+// Check for external database overrideif externalDsn := os.Getenv("DANGEROUS_EXTERNAL_DATABASE_DSN"); externalDsn != "" {log.Warn(diff --git a/docker-compose.example.yml b/docker-compose.example.ymlindex 9147a5df..938aad5f 100644--- a/docker-compose.example.yml+++ b/docker-compose.example.yml@@ -19,6 +19,21 @@ services:command: -p 5437shm_size: 10gb+ # Dedicated test DB so `make test` doesn't pollute the dev DB+ backend-db-test:+ image: postgres:17+ ports:+ - "5438:5438"+ environment:+ - POSTGRES_DB=${DEV_DB_NAME}+ - POSTGRES_USER=${DEV_DB_USERNAME}+ - POSTGRES_PASSWORD=${DEV_DB_PASSWORD}+ volumes:+ - ./pgdata-test:/var/lib/postgresql/data+ container_name: backend-db-test+ command: -p 5438+ shm_size: 10gb+# Valkey for cachingbackend-valkey:image: valkey/valkey:9.0.1-alpine