PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (verification): Create archive owner roles before pg_restore (issue #721)
Rostislav Dugin4fb5e6592ebb
diff --git a/agent/verification/e2e/fixture-gen/entrypoint.sh b/agent/verification/e2e/fixture-gen/entrypoint.shindex 3878f318..23742505 100644--- a/agent/verification/e2e/fixture-gen/entrypoint.sh+++ b/agent/verification/e2e/fixture-gen/entrypoint.sh@@ -65,7 +65,9 @@ done# (timescale/timescaledb:2.17.0-pg17), so the archive and the extension catalog# version match. It carries a hypertable spanning many chunks; restoring it# needs the agent's timescaledb_pre_restore / timescaledb_post_restore wrapping-# and single-threaded -j, or the _timescaledb_catalog restore fails.+# and single-threaded -j, or the _timescaledb_catalog restore fails. The+# hypertable and its retention policy belong to a non-bootstrap role, so the+# restore also needs that role recreated in the target before COPY.TS_IMAGE="timescale/timescaledb:2.17.0-pg17"TS_OUT="$ARTIFACTS/good-timescale-pg17.dump"@@ -88,15 +90,23 @@ fidocker exec -i "$CID" psql -U postgres -d postgres -v ON_ERROR_STOP=1 <<'SQL'CREATE EXTENSION IF NOT EXISTS timescaledb;+CREATE ROLE ts_app LOGIN;+GRANT CREATE ON SCHEMA public TO ts_app;+SET ROLE ts_app;CREATE TABLE public.sensor_data (time TIMESTAMPTZ NOT NULL,sensor_id int NOT NULL,temperature double precision NOT NULL);SELECT create_hypertable('public.sensor_data', 'time');+-- The retention policy lands in _timescaledb_config.bgw_job with owner = ts_app, a regrole+-- value the restore target must be able to resolve (issue #721). The interval is absurdly+-- long so the policy never drops a chunk the assertions count.+SELECT add_retention_policy('public.sensor_data', INTERVAL '1000 years');INSERT INTO public.sensor_data (time, sensor_id, temperature)SELECT ts, (random() * 10)::int, random() * 100FROM generate_series('2024-01-01'::timestamptz, '2024-03-01'::timestamptz, interval '1 hour') AS ts;+RESET ROLE;SQLdocker exec "$CID" pg_dump -Fc -U postgres postgres > "$TS_OUT"diff --git a/agent/verification/internal/features/restore/archiveownerroles.go b/agent/verification/internal/features/restore/archiveownerroles.gonew file mode 100644index 00000000..01742c58--- /dev/null+++ b/agent/verification/internal/features/restore/archiveownerroles.go@@ -0,0 +1,121 @@+package restore++import (+ "context"+ "fmt"+ "regexp"+ "strings"++ "github.com/jackc/pgx/v5"++ "databasus-verification-agent/internal/features/dbconn"+)++const bootstrapRole = "postgres"++var tocEntryPattern = regexp.MustCompile(`^\d+; \d+ \d+ \S+ .+ \S+$`)++// The throwaway target starts with only the bootstrap superuser, and pg_restore runs+// --no-owner --no-acl, so object ownership never needs these roles. Data does:+// TimescaleDB's _timescaledb_config.bgw_job.owner is a regrole column, and COPY resolves+// each role name to an OID as it loads, so a policy owned by an application role fails the+// whole restore with `role "..." does not exist` (issue #721). Coverage is the archive's TOC+// owners, which is every role the dump names as an owner — a regrole value pointing at a role+// that owns no archived object is out of reach and would still fail.+// The ensured role names are returned so the caller can log them under its job scope.+func (r *Restorer) EnsureArchiveOwnerRoles(+ ctx context.Context,+ exec ExecRunner,+ archivePath string,+ conn dbconn.Conn,+) ([]string, error) {+ roleNames, err := getArchiveOwnerRoleNames(ctx, exec, archivePath)+ if err != nil {+ return nil, err+ }++ if len(roleNames) == 0 {+ return nil, nil+ }++ if err := createMissingOwnerRoles(ctx, conn, roleNames); err != nil {+ return nil, err+ }++ return roleNames, nil+}++func getArchiveOwnerRoleNames(+ ctx context.Context,+ exec ExecRunner,+ archivePath string,+) ([]string, error) {+ result, err := exec.Exec(ctx, []string{"pg_restore", "-l", archivePath}, nil, nil)+ if err != nil {+ return nil, fmt.Errorf("pg_restore -l exec: %w", err)+ }++ // An unreadable archive is a verdict on the backup, not on this step, and the restore that+ // follows delivers it with the exit code the backend needs to mark the backup rejected.+ // Failing here instead would report no exit code, which the backend reads as agent+ // infrastructure trouble and retries forever.+ if result.ExitCode != 0 {+ return nil, nil+ }++ return parseTocOwnerRoleNames(result.Stdout), nil+}++// pg_restore -l prints one entry per line as "<id>; <tableoid> <oid> <desc> <schema> <tag>+// <owner>". Descriptions ("TABLE DATA") and tags ("add(integer, integer)") may contain+// spaces, but the owner is always last, so the trailing field is the only reliable anchor.+// An ownerless entry prints an empty trailing field, leaving the line ending in a space —+// hence the deliberate check before any trimming, without which the object tag would be+// mistaken for a role name.+func parseTocOwnerRoleNames(tocListing string) []string {+ seen := map[string]struct{}{}++ var roleNames []string++ for line := range strings.SplitSeq(tocListing, "\n") {+ line = strings.TrimRight(line, "\r")+ if line == "" || strings.HasPrefix(line, ";") || strings.HasSuffix(line, " ") {+ continue+ }++ if !tocEntryPattern.MatchString(line) {+ continue+ }++ fields := strings.Fields(line)++ roleName := fields[len(fields)-1]+ if roleName == bootstrapRole || strings.HasPrefix(roleName, "pg_") {+ continue+ }++ if _, isDup := seen[roleName]; isDup {+ continue+ }++ seen[roleName] = struct{}{}+ roleNames = append(roleNames, roleName)+ }++ return roleNames+}++// A role the target already carries (the bootstrap superuser aside, an image may ship its own)+// is not a conflict here, so each CREATE swallows duplicate_object rather than aborting.+func createMissingOwnerRoles(ctx context.Context, conn dbconn.Conn, roleNames []string) error {+ statements := make([]string, 0, len(roleNames))++ for _, roleName := range roleNames {+ statements = append(statements, fmt.Sprintf(+ "DO $$ BEGIN CREATE ROLE %s NOLOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$",+ pgx.Identifier{roleName}.Sanitize(),+ ))+ }++ return execStatements(ctx, conn, statements...)+}diff --git a/agent/verification/internal/features/restore/archiveownerroles_test.go b/agent/verification/internal/features/restore/archiveownerroles_test.gonew file mode 100644index 00000000..89c81236--- /dev/null+++ b/agent/verification/internal/features/restore/archiveownerroles_test.go@@ -0,0 +1,73 @@+package restore++import (+ "errors"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "databasus-verification-agent/internal/testutil"+)++// Verbatim `pg_restore -l` output shapes: header comments, a multi-word desc, a tag holding+// a space, an ownerless entry (trailing space after the tag), reserved and bootstrap roles.+const tocListing = ";\n" ++ "; Archive created at 2026-01-01 00:00:00 UTC\n" ++ "; dbname: postgres\n" ++ ";\n" ++ "4; 3079 16388 EXTENSION - timescaledb \n" ++ "215; 1255 16400 FUNCTION public add(integer, integer) ts_app\n" ++ "251; 1259 16404 TABLE public sensor_data ts_app\n" ++ "3057; 0 16404 TABLE DATA public sensor_data ts_app\n" ++ "260; 1259 16410 TABLE public audit_log reporting\n" ++ "2615; 2200 2200 SCHEMA - public postgres\n" ++ "270; 1259 16420 TABLE public masked pg_monitor\n"++func Test_ParseTocOwnerRoleNames_WhenTocHasOwners_ReturnsUniqueNonReservedNames(t *testing.T) {+ roleNames := parseTocOwnerRoleNames(tocListing)++ assert.Equal(t, []string{"ts_app", "reporting"}, roleNames)+}++func Test_ParseTocOwnerRoleNames_WhenListingIsEmpty_ReturnsNothing(t *testing.T) {+ assert.Empty(t, parseTocOwnerRoleNames(""))+}++func Test_ParseTocOwnerRoleNames_WhenEntryHasNoOwner_SkipsTheTag(t *testing.T) {+ roleNames := parseTocOwnerRoleNames("4; 3079 16388 EXTENSION - timescaledb \n")++ assert.Empty(t, roleNames, "an ownerless entry must not turn its tag into a role name")+}++func Test_EnsureArchiveOwnerRoles_WhenArchiveHasNoOwners_SkipsTheDatabase(t *testing.T) {+ exec := &fakeExecRunner{result: ExecResult{ExitCode: 0, Stdout: ";\n"}}+ r := NewRestorer(testutil.DiscardLogger())++ ensuredRoleNames, err := r.EnsureArchiveOwnerRoles(t.Context(), exec, "/tmp/x.dump", testConn())++ require.NoError(t, err)+ assert.Empty(t, ensuredRoleNames)+ require.Len(t, exec.recorded, 1)+ assert.Equal(t, []string{"pg_restore", "-l", "/tmp/x.dump"}, exec.recorded[0].cmd)+}++func Test_EnsureArchiveOwnerRoles_WhenListingFails_ReturnsError(t *testing.T) {+ exec := &fakeExecRunner{err: errors.New("exec attach failed")}+ r := NewRestorer(testutil.DiscardLogger())++ _, err := r.EnsureArchiveOwnerRoles(t.Context(), exec, "/tmp/x.dump", testConn())++ require.ErrorContains(t, err, "pg_restore -l exec")+}++func Test_EnsureArchiveOwnerRoles_WhenArchiveIsUnreadable_YieldsToPgRestoreVerdict(t *testing.T) {+ exec := &fakeExecRunner{result: ExecResult{ExitCode: 1, Stderr: "not a valid archive"}}+ r := NewRestorer(testutil.DiscardLogger())++ ensuredRoleNames, err := r.EnsureArchiveOwnerRoles(t.Context(), exec, "/tmp/x.dump", testConn())++ require.NoError(t, err,+ "a corrupt archive must reach pg_restore, whose exit code lets the backend reject the backup")+ assert.Empty(t, ensuredRoleNames)+}diff --git a/agent/verification/internal/features/runner/concurrent_restores_integration_test.go b/agent/verification/internal/features/runner/concurrent_restores_integration_test.goindex 8a154406..e9552b6f 100644--- a/agent/verification/internal/features/runner/concurrent_restores_integration_test.go+++ b/agent/verification/internal/features/runner/concurrent_restores_integration_test.go@@ -192,6 +192,12 @@ func (r *barrierRestorer) RunPgRestore(return restore.Result{PgRestoreExitCode: 0, DurationMs: 1}, nil}+func (r *barrierRestorer) EnsureArchiveOwnerRoles(+ _ context.Context, _ restore.ExecRunner, _ string, _ dbconn.Conn,+) ([]string, error) {+ return nil, nil+}+func (r *barrierRestorer) RunTimescalePreRestore(context.Context, dbconn.Conn) error { return nil }func (r *barrierRestorer) RunTimescalePostRestore(context.Context, dbconn.Conn) error { return nil }diff --git a/agent/verification/internal/features/runner/fakes_test.go b/agent/verification/internal/features/runner/fakes_test.goindex 95b67c7c..b7b4363c 100644--- a/agent/verification/internal/features/runner/fakes_test.go+++ b/agent/verification/internal/features/runner/fakes_test.go@@ -129,15 +129,16 @@ func (s *fakeSpawner) Spawn(_ context.Context, _ SpawnRequest) (JobContainer, er}type fakeRestorer struct {- stageErr error- runResult restore.Result- runErr error- runBlocks bool+ stageErr error+ ensureRolesErr error+ runResult restore.Result+ runErr error+ runBlocks boolrunEntered atomic.BoolrunCtxCancelled atomic.Bool- // calls records the restore-related call order ("pre", "restore", "post") for a single job.+ // calls records the restore-related call order ("roles", "pre", "restore", "post") for a single job.calls []string// runParallelJobs records the -j value RunPgRestore was invoked with.runParallelJobs int@@ -170,6 +171,18 @@ func (r *fakeRestorer) RunPgRestore(return r.runResult, r.runErr}+func (r *fakeRestorer) EnsureArchiveOwnerRoles(+ _ context.Context, _ restore.ExecRunner, _ string, _ dbconn.Conn,+) ([]string, error) {+ r.calls = append(r.calls, "roles")++ if r.ensureRolesErr != nil {+ return nil, r.ensureRolesErr+ }++ return []string{"ts_app"}, nil+}+func (r *fakeRestorer) RunTimescalePreRestore(context.Context, dbconn.Conn) error {r.calls = append(r.calls, "pre")diff --git a/agent/verification/internal/features/runner/interfaces.go b/agent/verification/internal/features/runner/interfaces.goindex f78de084..5766ef54 100644--- a/agent/verification/internal/features/runner/interfaces.go+++ b/agent/verification/internal/features/runner/interfaces.go@@ -39,6 +39,12 @@ type Restorer interface {conn dbconn.Conn,parallelJobs int,) (restore.Result, error)+ EnsureArchiveOwnerRoles(+ ctx context.Context,+ exec restore.ExecRunner,+ archivePath string,+ conn dbconn.Conn,+ ) ([]string, error)RunTimescalePreRestore(ctx context.Context, conn dbconn.Conn) errorRunTimescalePostRestore(ctx context.Context, conn dbconn.Conn) error}diff --git a/agent/verification/internal/features/runner/runner.go b/agent/verification/internal/features/runner/runner.goindex 7e3ab3ee..3ea64d84 100644--- a/agent/verification/internal/features/runner/runner.go+++ b/agent/verification/internal/features/runner/runner.go@@ -234,6 +234,25 @@ func (r *Runner) executeJob(ctx context.Context, job *api.JobAssignment) {return}+ archiveOwnerRoleNames, err := r.restorer.EnsureArchiveOwnerRoles(+ jobCtx, jobContainer, archivePath, jobContainer.GetVerifierConn())+ if err != nil {+ if jobCtx.Err() != nil {+ return+ }++ r.reportFailure(ctx, job.VerificationID, nil,+ fmt.Sprintf("ensure archive owner roles: %v", err), runLogger)++ return+ }++ if len(archiveOwnerRoleNames) > 0 {+ runLogger.Info(fmt.Sprintf(+ "archive owner roles present in restore target: roles=%s",+ strings.Join(archiveOwnerRoleNames, ",")))+ }+parallelJobs := min(maxParallelRestoreJobs, r.capacity.CPUPerJob)// TimescaleDB restores single-threaded and in restoring mode. Single-threaded because paralleldiff --git a/agent/verification/internal/features/runner/runner_test.go b/agent/verification/internal/features/runner/runner_test.goindex 5596c4be..93688ab7 100644--- a/agent/verification/internal/features/runner/runner_test.go+++ b/agent/verification/internal/features/runner/runner_test.go@@ -316,7 +316,7 @@ func Test_ExecuteJob_WhenTimescaleJobSucceeds_RunsPreRestoreThenRestoreThenPostRreport, ok := apiClient.lastReport()require.True(t, ok)assert.Equal(t, api.VerificationStatusCompleted, report.Status)- assert.Equal(t, []string{"pre", "restore", "post"}, restorer.calls,+ assert.Equal(t, []string{"roles", "pre", "restore", "post"}, restorer.calls,"timescaledb restore must enter restoring mode, restore, then leave it")assert.Equal(t, 1, restorer.runParallelJobs,"timescaledb restore must be single-threaded to keep _timescaledb_catalog FK order")@@ -331,12 +331,27 @@ func Test_ExecuteJob_WhenNonTimescaleJob_DoesNotRunTimescaleHooks(t *testing.T)r.executeJob(t.Context(), postgresJob())- assert.Equal(t, []string{"restore"}, restorer.calls,+ assert.Equal(t, []string{"roles", "restore"}, restorer.calls,"a non-timescaledb restore must not run the timescaledb hooks")assert.Equal(t, 2, restorer.runParallelJobs,"a non-timescaledb restore keeps parallel jobs (CPUPerJob)")}+func Test_ExecuteJob_WhenOwnerRoleCreationFails_ReportsFailedWithoutRestoring(t *testing.T) {+ apiClient := &fakeAPI{downloadBody: []byte("ARCHIVE")}+ restorer := &fakeRestorer{ensureRolesErr: errors.New("connect restore target: refused")}+ r := newTestRunner(apiClient, okSpawner(), restorer, &fakeStats{}, newFakeRegistrar())++ r.executeJob(t.Context(), postgresJob())++ report, ok := apiClient.lastReport()+ require.True(t, ok)+ assert.Equal(t, api.VerificationStatusFailed, report.Status)+ assert.Nil(t, report.PgRestoreExitCode,+ "role setup is agent infrastructure, so the backup must not be blamed")+ assert.Equal(t, []string{"roles"}, restorer.calls)+}+func Test_ExecuteJob_WhenAbortedBeforeFailedReport_DoesNotReport(t *testing.T) {apiClient := &fakeAPI{}spawner := &fakeSpawner{err: errors.New("spawn failed")}