Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (verification): Create archive owner roles before pg_restore (issue #721)

Rostislav Dugin4fb5e6592ebb
diff --git a/agent/verification/e2e/fixture-gen/entrypoint.sh b/agent/verification/e2e/fixture-gen/entrypoint.sh
index 3878f318..23742505 100644
--- a/agent/verification/e2e/fixture-gen/entrypoint.sh
+++ b/agent/verification/e2e/fixture-gen/entrypoint.sh
@@ -65,7 +65,9 @@ done
# (timescale/timescaledb:2.17.0-pg17), so the archive and the extension catalog
# version match. It carries a hypertable spanning many chunks; restoring it
# needs the agent's timescaledb_pre_restore / timescaledb_post_restore wrapping
-# and single-threaded -j, or the _timescaledb_catalog restore fails.
+# and single-threaded -j, or the _timescaledb_catalog restore fails. The
+# hypertable and its retention policy belong to a non-bootstrap role, so the
+# restore also needs that role recreated in the target before COPY.
TS_IMAGE="timescale/timescaledb:2.17.0-pg17"
TS_OUT="$ARTIFACTS/good-timescale-pg17.dump"
@@ -88,15 +90,23 @@ fi
docker exec -i "$CID" psql -U postgres -d postgres -v ON_ERROR_STOP=1 <<'SQL'
CREATE EXTENSION IF NOT EXISTS timescaledb;
+CREATE ROLE ts_app LOGIN;
+GRANT CREATE ON SCHEMA public TO ts_app;
+SET ROLE ts_app;
CREATE TABLE public.sensor_data (
time TIMESTAMPTZ NOT NULL,
sensor_id int NOT NULL,
temperature double precision NOT NULL
);
SELECT create_hypertable('public.sensor_data', 'time');
+-- The retention policy lands in _timescaledb_config.bgw_job with owner = ts_app, a regrole
+-- value the restore target must be able to resolve (issue #721). The interval is absurdly
+-- long so the policy never drops a chunk the assertions count.
+SELECT add_retention_policy('public.sensor_data', INTERVAL '1000 years');
INSERT INTO public.sensor_data (time, sensor_id, temperature)
SELECT ts, (random() * 10)::int, random() * 100
FROM generate_series('2024-01-01'::timestamptz, '2024-03-01'::timestamptz, interval '1 hour') AS ts;
+RESET ROLE;
SQL
docker exec "$CID" pg_dump -Fc -U postgres postgres > "$TS_OUT"
diff --git a/agent/verification/internal/features/restore/archiveownerroles.go b/agent/verification/internal/features/restore/archiveownerroles.go
new file mode 100644
index 00000000..01742c58
--- /dev/null
+++ b/agent/verification/internal/features/restore/archiveownerroles.go
@@ -0,0 +1,121 @@
+package restore
+
+import (
+ "context"
+ "fmt"
+ "regexp"
+ "strings"
+
+ "github.com/jackc/pgx/v5"
+
+ "databasus-verification-agent/internal/features/dbconn"
+)
+
+const bootstrapRole = "postgres"
+
+var tocEntryPattern = regexp.MustCompile(`^\d+; \d+ \d+ \S+ .+ \S+$`)
+
+// The throwaway target starts with only the bootstrap superuser, and pg_restore runs
+// --no-owner --no-acl, so object ownership never needs these roles. Data does:
+// TimescaleDB's _timescaledb_config.bgw_job.owner is a regrole column, and COPY resolves
+// each role name to an OID as it loads, so a policy owned by an application role fails the
+// whole restore with `role "..." does not exist` (issue #721). Coverage is the archive's TOC
+// owners, which is every role the dump names as an owner — a regrole value pointing at a role
+// that owns no archived object is out of reach and would still fail.
+// The ensured role names are returned so the caller can log them under its job scope.
+func (r *Restorer) EnsureArchiveOwnerRoles(
+ ctx context.Context,
+ exec ExecRunner,
+ archivePath string,
+ conn dbconn.Conn,
+) ([]string, error) {
+ roleNames, err := getArchiveOwnerRoleNames(ctx, exec, archivePath)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(roleNames) == 0 {
+ return nil, nil
+ }
+
+ if err := createMissingOwnerRoles(ctx, conn, roleNames); err != nil {
+ return nil, err
+ }
+
+ return roleNames, nil
+}
+
+func getArchiveOwnerRoleNames(
+ ctx context.Context,
+ exec ExecRunner,
+ archivePath string,
+) ([]string, error) {
+ result, err := exec.Exec(ctx, []string{"pg_restore", "-l", archivePath}, nil, nil)
+ if err != nil {
+ return nil, fmt.Errorf("pg_restore -l exec: %w", err)
+ }
+
+ // An unreadable archive is a verdict on the backup, not on this step, and the restore that
+ // follows delivers it with the exit code the backend needs to mark the backup rejected.
+ // Failing here instead would report no exit code, which the backend reads as agent
+ // infrastructure trouble and retries forever.
+ if result.ExitCode != 0 {
+ return nil, nil
+ }
+
+ return parseTocOwnerRoleNames(result.Stdout), nil
+}
+
+// pg_restore -l prints one entry per line as "<id>; <tableoid> <oid> <desc> <schema> <tag>
+// <owner>". Descriptions ("TABLE DATA") and tags ("add(integer, integer)") may contain
+// spaces, but the owner is always last, so the trailing field is the only reliable anchor.
+// An ownerless entry prints an empty trailing field, leaving the line ending in a space —
+// hence the deliberate check before any trimming, without which the object tag would be
+// mistaken for a role name.
+func parseTocOwnerRoleNames(tocListing string) []string {
+ seen := map[string]struct{}{}
+
+ var roleNames []string
+
+ for line := range strings.SplitSeq(tocListing, "\n") {
+ line = strings.TrimRight(line, "\r")
+ if line == "" || strings.HasPrefix(line, ";") || strings.HasSuffix(line, " ") {
+ continue
+ }
+
+ if !tocEntryPattern.MatchString(line) {
+ continue
+ }
+
+ fields := strings.Fields(line)
+
+ roleName := fields[len(fields)-1]
+ if roleName == bootstrapRole || strings.HasPrefix(roleName, "pg_") {
+ continue
+ }
+
+ if _, isDup := seen[roleName]; isDup {
+ continue
+ }
+
+ seen[roleName] = struct{}{}
+ roleNames = append(roleNames, roleName)
+ }
+
+ return roleNames
+}
+
+// A role the target already carries (the bootstrap superuser aside, an image may ship its own)
+// is not a conflict here, so each CREATE swallows duplicate_object rather than aborting.
+func createMissingOwnerRoles(ctx context.Context, conn dbconn.Conn, roleNames []string) error {
+ statements := make([]string, 0, len(roleNames))
+
+ for _, roleName := range roleNames {
+ statements = append(statements, fmt.Sprintf(
+ "DO $$ BEGIN CREATE ROLE %s NOLOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$",
+ pgx.Identifier{roleName}.Sanitize(),
+ ))
+ }
+
+ return execStatements(ctx, conn, statements...)
+}
diff --git a/agent/verification/internal/features/restore/archiveownerroles_test.go b/agent/verification/internal/features/restore/archiveownerroles_test.go
new file mode 100644
index 00000000..89c81236
--- /dev/null
+++ b/agent/verification/internal/features/restore/archiveownerroles_test.go
@@ -0,0 +1,73 @@
+package restore
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-verification-agent/internal/testutil"
+)
+
+// Verbatim `pg_restore -l` output shapes: header comments, a multi-word desc, a tag holding
+// a space, an ownerless entry (trailing space after the tag), reserved and bootstrap roles.
+const tocListing = ";\n" +
+ "; Archive created at 2026-01-01 00:00:00 UTC\n" +
+ "; dbname: postgres\n" +
+ ";\n" +
+ "4; 3079 16388 EXTENSION - timescaledb \n" +
+ "215; 1255 16400 FUNCTION public add(integer, integer) ts_app\n" +
+ "251; 1259 16404 TABLE public sensor_data ts_app\n" +
+ "3057; 0 16404 TABLE DATA public sensor_data ts_app\n" +
+ "260; 1259 16410 TABLE public audit_log reporting\n" +
+ "2615; 2200 2200 SCHEMA - public postgres\n" +
+ "270; 1259 16420 TABLE public masked pg_monitor\n"
+
+func Test_ParseTocOwnerRoleNames_WhenTocHasOwners_ReturnsUniqueNonReservedNames(t *testing.T) {
+ roleNames := parseTocOwnerRoleNames(tocListing)
+
+ assert.Equal(t, []string{"ts_app", "reporting"}, roleNames)
+}
+
+func Test_ParseTocOwnerRoleNames_WhenListingIsEmpty_ReturnsNothing(t *testing.T) {
+ assert.Empty(t, parseTocOwnerRoleNames(""))
+}
+
+func Test_ParseTocOwnerRoleNames_WhenEntryHasNoOwner_SkipsTheTag(t *testing.T) {
+ roleNames := parseTocOwnerRoleNames("4; 3079 16388 EXTENSION - timescaledb \n")
+
+ assert.Empty(t, roleNames, "an ownerless entry must not turn its tag into a role name")
+}
+
+func Test_EnsureArchiveOwnerRoles_WhenArchiveHasNoOwners_SkipsTheDatabase(t *testing.T) {
+ exec := &fakeExecRunner{result: ExecResult{ExitCode: 0, Stdout: ";\n"}}
+ r := NewRestorer(testutil.DiscardLogger())
+
+ ensuredRoleNames, err := r.EnsureArchiveOwnerRoles(t.Context(), exec, "/tmp/x.dump", testConn())
+
+ require.NoError(t, err)
+ assert.Empty(t, ensuredRoleNames)
+ require.Len(t, exec.recorded, 1)
+ assert.Equal(t, []string{"pg_restore", "-l", "/tmp/x.dump"}, exec.recorded[0].cmd)
+}
+
+func Test_EnsureArchiveOwnerRoles_WhenListingFails_ReturnsError(t *testing.T) {
+ exec := &fakeExecRunner{err: errors.New("exec attach failed")}
+ r := NewRestorer(testutil.DiscardLogger())
+
+ _, err := r.EnsureArchiveOwnerRoles(t.Context(), exec, "/tmp/x.dump", testConn())
+
+ require.ErrorContains(t, err, "pg_restore -l exec")
+}
+
+func Test_EnsureArchiveOwnerRoles_WhenArchiveIsUnreadable_YieldsToPgRestoreVerdict(t *testing.T) {
+ exec := &fakeExecRunner{result: ExecResult{ExitCode: 1, Stderr: "not a valid archive"}}
+ r := NewRestorer(testutil.DiscardLogger())
+
+ ensuredRoleNames, err := r.EnsureArchiveOwnerRoles(t.Context(), exec, "/tmp/x.dump", testConn())
+
+ require.NoError(t, err,
+ "a corrupt archive must reach pg_restore, whose exit code lets the backend reject the backup")
+ assert.Empty(t, ensuredRoleNames)
+}
diff --git a/agent/verification/internal/features/runner/concurrent_restores_integration_test.go b/agent/verification/internal/features/runner/concurrent_restores_integration_test.go
index 8a154406..e9552b6f 100644
--- a/agent/verification/internal/features/runner/concurrent_restores_integration_test.go
+++ b/agent/verification/internal/features/runner/concurrent_restores_integration_test.go
@@ -192,6 +192,12 @@ func (r *barrierRestorer) RunPgRestore(
return restore.Result{PgRestoreExitCode: 0, DurationMs: 1}, nil
}
+func (r *barrierRestorer) EnsureArchiveOwnerRoles(
+ _ context.Context, _ restore.ExecRunner, _ string, _ dbconn.Conn,
+) ([]string, error) {
+ return nil, nil
+}
+
func (r *barrierRestorer) RunTimescalePreRestore(context.Context, dbconn.Conn) error { return nil }
func (r *barrierRestorer) RunTimescalePostRestore(context.Context, dbconn.Conn) error { return nil }
diff --git a/agent/verification/internal/features/runner/fakes_test.go b/agent/verification/internal/features/runner/fakes_test.go
index 95b67c7c..b7b4363c 100644
--- a/agent/verification/internal/features/runner/fakes_test.go
+++ b/agent/verification/internal/features/runner/fakes_test.go
@@ -129,15 +129,16 @@ func (s *fakeSpawner) Spawn(_ context.Context, _ SpawnRequest) (JobContainer, er
}
type fakeRestorer struct {
- stageErr error
- runResult restore.Result
- runErr error
- runBlocks bool
+ stageErr error
+ ensureRolesErr error
+ runResult restore.Result
+ runErr error
+ runBlocks bool
runEntered atomic.Bool
runCtxCancelled atomic.Bool
- // calls records the restore-related call order ("pre", "restore", "post") for a single job.
+ // calls records the restore-related call order ("roles", "pre", "restore", "post") for a single job.
calls []string
// runParallelJobs records the -j value RunPgRestore was invoked with.
runParallelJobs int
@@ -170,6 +171,18 @@ func (r *fakeRestorer) RunPgRestore(
return r.runResult, r.runErr
}
+func (r *fakeRestorer) EnsureArchiveOwnerRoles(
+ _ context.Context, _ restore.ExecRunner, _ string, _ dbconn.Conn,
+) ([]string, error) {
+ r.calls = append(r.calls, "roles")
+
+ if r.ensureRolesErr != nil {
+ return nil, r.ensureRolesErr
+ }
+
+ return []string{"ts_app"}, nil
+}
+
func (r *fakeRestorer) RunTimescalePreRestore(context.Context, dbconn.Conn) error {
r.calls = append(r.calls, "pre")
diff --git a/agent/verification/internal/features/runner/interfaces.go b/agent/verification/internal/features/runner/interfaces.go
index f78de084..5766ef54 100644
--- a/agent/verification/internal/features/runner/interfaces.go
+++ b/agent/verification/internal/features/runner/interfaces.go
@@ -39,6 +39,12 @@ type Restorer interface {
conn dbconn.Conn,
parallelJobs int,
) (restore.Result, error)
+ EnsureArchiveOwnerRoles(
+ ctx context.Context,
+ exec restore.ExecRunner,
+ archivePath string,
+ conn dbconn.Conn,
+ ) ([]string, error)
RunTimescalePreRestore(ctx context.Context, conn dbconn.Conn) error
RunTimescalePostRestore(ctx context.Context, conn dbconn.Conn) error
}
diff --git a/agent/verification/internal/features/runner/runner.go b/agent/verification/internal/features/runner/runner.go
index 7e3ab3ee..3ea64d84 100644
--- a/agent/verification/internal/features/runner/runner.go
+++ b/agent/verification/internal/features/runner/runner.go
@@ -234,6 +234,25 @@ func (r *Runner) executeJob(ctx context.Context, job *api.JobAssignment) {
return
}
+ archiveOwnerRoleNames, err := r.restorer.EnsureArchiveOwnerRoles(
+ jobCtx, jobContainer, archivePath, jobContainer.GetVerifierConn())
+ if err != nil {
+ if jobCtx.Err() != nil {
+ return
+ }
+
+ r.reportFailure(ctx, job.VerificationID, nil,
+ fmt.Sprintf("ensure archive owner roles: %v", err), runLogger)
+
+ return
+ }
+
+ if len(archiveOwnerRoleNames) > 0 {
+ runLogger.Info(fmt.Sprintf(
+ "archive owner roles present in restore target: roles=%s",
+ strings.Join(archiveOwnerRoleNames, ",")))
+ }
+
parallelJobs := min(maxParallelRestoreJobs, r.capacity.CPUPerJob)
// TimescaleDB restores single-threaded and in restoring mode. Single-threaded because parallel
diff --git a/agent/verification/internal/features/runner/runner_test.go b/agent/verification/internal/features/runner/runner_test.go
index 5596c4be..93688ab7 100644
--- a/agent/verification/internal/features/runner/runner_test.go
+++ b/agent/verification/internal/features/runner/runner_test.go
@@ -316,7 +316,7 @@ func Test_ExecuteJob_WhenTimescaleJobSucceeds_RunsPreRestoreThenRestoreThenPostR
report, ok := apiClient.lastReport()
require.True(t, ok)
assert.Equal(t, api.VerificationStatusCompleted, report.Status)
- assert.Equal(t, []string{"pre", "restore", "post"}, restorer.calls,
+ assert.Equal(t, []string{"roles", "pre", "restore", "post"}, restorer.calls,
"timescaledb restore must enter restoring mode, restore, then leave it")
assert.Equal(t, 1, restorer.runParallelJobs,
"timescaledb restore must be single-threaded to keep _timescaledb_catalog FK order")
@@ -331,12 +331,27 @@ func Test_ExecuteJob_WhenNonTimescaleJob_DoesNotRunTimescaleHooks(t *testing.T)
r.executeJob(t.Context(), postgresJob())
- assert.Equal(t, []string{"restore"}, restorer.calls,
+ assert.Equal(t, []string{"roles", "restore"}, restorer.calls,
"a non-timescaledb restore must not run the timescaledb hooks")
assert.Equal(t, 2, restorer.runParallelJobs,
"a non-timescaledb restore keeps parallel jobs (CPUPerJob)")
}
+func Test_ExecuteJob_WhenOwnerRoleCreationFails_ReportsFailedWithoutRestoring(t *testing.T) {
+ apiClient := &fakeAPI{downloadBody: []byte("ARCHIVE")}
+ restorer := &fakeRestorer{ensureRolesErr: errors.New("connect restore target: refused")}
+ r := newTestRunner(apiClient, okSpawner(), restorer, &fakeStats{}, newFakeRegistrar())
+
+ r.executeJob(t.Context(), postgresJob())
+
+ report, ok := apiClient.lastReport()
+ require.True(t, ok)
+ assert.Equal(t, api.VerificationStatusFailed, report.Status)
+ assert.Nil(t, report.PgRestoreExitCode,
+ "role setup is agent infrastructure, so the backup must not be blamed")
+ assert.Equal(t, []string{"roles"}, restorer.calls)
+}
+
func Test_ExecuteJob_WhenAbortedBeforeFailedReport_DoesNotReport(t *testing.T) {
apiClient := &fakeAPI{}
spawner := &fakeSpawner{err: errors.New("spawn failed")}