Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (postgresql): Update bundled pg_dump past the 18.1 sequence-value bug (issue #725)

Rostislav Duginddb631ef1afc
diff --git a/assets/tools/README.md b/assets/tools/README.md
index 3daf0a72..9ae491f3 100644
--- a/assets/tools/README.md
+++ b/assets/tools/README.md
@@ -24,6 +24,23 @@ assets/tools/<arch>/
| `linux` / `amd64` | `x64` | ~160 MB |
| `linux` / `arm64` | `arm` | ~125 MB |
+Bundled PostgreSQL minors (identical on both arches, Debian bookworm pgdg):
+
+| major | minor | source package |
+|-------|-------|-----------------------------------|
+| 12 | 12.22 | `12.22-3.pgdg12+1` (final, EOL) |
+| 13 | 13.23 | `13.23-1.pgdg12+1` (final, EOL) |
+| 14 | 14.23 | `14.23-1.pgdg12+1` |
+| 15 | 15.18 | `15.18-1.pgdg12+1` |
+| 16 | 16.14 | `16.14-1.pgdg12+1` |
+| 17 | 17.10 | `17.10-1.pgdg12+1` |
+| 18 | 18.4 | `18.4-1.pgdg12+1` |
+
+Majors 12 and 13 are past upstream EOL, so those minors are the last ones that
+will ever exist. Keep every binary within one major on the same package build —
+a `pg_dump` older than its sibling `pg_basebackup` is how issue #725 (pg_dump
+18.1 silently emitting wrong sequence values) survived unnoticed.
+
Notes:
- MySQL `5.7` is amd64-only — `arm/mysql/mysql-5.7/` is intentionally absent.
- MariaDB ships two client versions: legacy `10.6` (for MariaDB servers 5.5 / 10.1) and modern `12.1` (for 10.2+). The mapping lives in `tools/mariadb.go`.
@@ -31,4 +48,7 @@ Notes:
To refresh a tool set, drop the corresponding `bin/` contents in place
and commit. There are no install scripts in this directory; binaries
-are sourced from the upstream vendor downloads.
+are sourced from the upstream vendor downloads. For PostgreSQL that means
+`postgresql-client-<major>` from `apt.postgresql.org` for `bookworm`, both
+`amd64` and `arm64`, matching the `debian:bookworm-slim` runtime base in the
+Dockerfile. Update the minor table above in the same commit.
diff --git a/assets/tools/arm/postgresql/postgresql-14/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-14/bin/pg_dump
index 7c6bcade..9b7d7b6c 100755
Binary files a/assets/tools/arm/postgresql/postgresql-14/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-14/bin/pg_dump differ
diff --git a/assets/tools/arm/postgresql/postgresql-14/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-14/bin/pg_restore
index 6df17ae2..3f3df196 100755
Binary files a/assets/tools/arm/postgresql/postgresql-14/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-14/bin/pg_restore differ
diff --git a/assets/tools/arm/postgresql/postgresql-14/bin/psql b/assets/tools/arm/postgresql/postgresql-14/bin/psql
index 8616815a..6e999006 100755
Binary files a/assets/tools/arm/postgresql/postgresql-14/bin/psql and b/assets/tools/arm/postgresql/postgresql-14/bin/psql differ
diff --git a/assets/tools/arm/postgresql/postgresql-15/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-15/bin/pg_dump
index 8eecace9..4ba39734 100755
Binary files a/assets/tools/arm/postgresql/postgresql-15/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-15/bin/pg_dump differ
diff --git a/assets/tools/arm/postgresql/postgresql-15/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-15/bin/pg_restore
index 6ad5e4be..1cfa8839 100755
Binary files a/assets/tools/arm/postgresql/postgresql-15/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-15/bin/pg_restore differ
diff --git a/assets/tools/arm/postgresql/postgresql-15/bin/psql b/assets/tools/arm/postgresql/postgresql-15/bin/psql
index ce119392..ba0f0ecd 100755
Binary files a/assets/tools/arm/postgresql/postgresql-15/bin/psql and b/assets/tools/arm/postgresql/postgresql-15/bin/psql differ
diff --git a/assets/tools/arm/postgresql/postgresql-16/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-16/bin/pg_dump
index 88ebd978..6ca72ef3 100755
Binary files a/assets/tools/arm/postgresql/postgresql-16/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-16/bin/pg_dump differ
diff --git a/assets/tools/arm/postgresql/postgresql-16/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-16/bin/pg_restore
index eee24bb1..c7a43811 100755
Binary files a/assets/tools/arm/postgresql/postgresql-16/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-16/bin/pg_restore differ
diff --git a/assets/tools/arm/postgresql/postgresql-16/bin/psql b/assets/tools/arm/postgresql/postgresql-16/bin/psql
index e96c287b..019c475a 100755
Binary files a/assets/tools/arm/postgresql/postgresql-16/bin/psql and b/assets/tools/arm/postgresql/postgresql-16/bin/psql differ
diff --git a/assets/tools/arm/postgresql/postgresql-17/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-17/bin/pg_dump
index 2eb8f0eb..1db9e773 100755
Binary files a/assets/tools/arm/postgresql/postgresql-17/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-17/bin/pg_dump differ
diff --git a/assets/tools/arm/postgresql/postgresql-17/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-17/bin/pg_restore
index 87a71cc6..e1f0a0a2 100755
Binary files a/assets/tools/arm/postgresql/postgresql-17/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-17/bin/pg_restore differ
diff --git a/assets/tools/arm/postgresql/postgresql-17/bin/psql b/assets/tools/arm/postgresql/postgresql-17/bin/psql
index d981a9ce..0e078476 100755
Binary files a/assets/tools/arm/postgresql/postgresql-17/bin/psql and b/assets/tools/arm/postgresql/postgresql-17/bin/psql differ
diff --git a/assets/tools/arm/postgresql/postgresql-18/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-18/bin/pg_dump
index e770af77..f1554f6f 100755
Binary files a/assets/tools/arm/postgresql/postgresql-18/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-18/bin/pg_dump differ
diff --git a/assets/tools/arm/postgresql/postgresql-18/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-18/bin/pg_restore
index d4a58fe1..d0609070 100755
Binary files a/assets/tools/arm/postgresql/postgresql-18/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-18/bin/pg_restore differ
diff --git a/assets/tools/arm/postgresql/postgresql-18/bin/psql b/assets/tools/arm/postgresql/postgresql-18/bin/psql
index db4c8e27..17bff46f 100755
Binary files a/assets/tools/arm/postgresql/postgresql-18/bin/psql and b/assets/tools/arm/postgresql/postgresql-18/bin/psql differ
diff --git a/assets/tools/x64/postgresql/postgresql-14/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-14/bin/pg_dump
index 955b2435..ff1bc4ce 100755
Binary files a/assets/tools/x64/postgresql/postgresql-14/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-14/bin/pg_dump differ
diff --git a/assets/tools/x64/postgresql/postgresql-14/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-14/bin/pg_restore
index 355c3159..272c5fa4 100755
Binary files a/assets/tools/x64/postgresql/postgresql-14/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-14/bin/pg_restore differ
diff --git a/assets/tools/x64/postgresql/postgresql-14/bin/psql b/assets/tools/x64/postgresql/postgresql-14/bin/psql
index 48192354..8c89979a 100755
Binary files a/assets/tools/x64/postgresql/postgresql-14/bin/psql and b/assets/tools/x64/postgresql/postgresql-14/bin/psql differ
diff --git a/assets/tools/x64/postgresql/postgresql-15/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-15/bin/pg_dump
index 8881dabb..7b1c271d 100755
Binary files a/assets/tools/x64/postgresql/postgresql-15/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-15/bin/pg_dump differ
diff --git a/assets/tools/x64/postgresql/postgresql-15/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-15/bin/pg_restore
index cbd44ba7..a8546a51 100755
Binary files a/assets/tools/x64/postgresql/postgresql-15/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-15/bin/pg_restore differ
diff --git a/assets/tools/x64/postgresql/postgresql-15/bin/psql b/assets/tools/x64/postgresql/postgresql-15/bin/psql
index 260408d6..e3bb8add 100755
Binary files a/assets/tools/x64/postgresql/postgresql-15/bin/psql and b/assets/tools/x64/postgresql/postgresql-15/bin/psql differ
diff --git a/assets/tools/x64/postgresql/postgresql-16/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-16/bin/pg_dump
index 8d20464a..34e81ed9 100755
Binary files a/assets/tools/x64/postgresql/postgresql-16/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-16/bin/pg_dump differ
diff --git a/assets/tools/x64/postgresql/postgresql-16/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-16/bin/pg_restore
index 929e15aa..fa073ec1 100755
Binary files a/assets/tools/x64/postgresql/postgresql-16/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-16/bin/pg_restore differ
diff --git a/assets/tools/x64/postgresql/postgresql-16/bin/psql b/assets/tools/x64/postgresql/postgresql-16/bin/psql
index 3855edd9..e4982ebb 100755
Binary files a/assets/tools/x64/postgresql/postgresql-16/bin/psql and b/assets/tools/x64/postgresql/postgresql-16/bin/psql differ
diff --git a/assets/tools/x64/postgresql/postgresql-17/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-17/bin/pg_dump
index 51a235d7..b650f887 100755
Binary files a/assets/tools/x64/postgresql/postgresql-17/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-17/bin/pg_dump differ
diff --git a/assets/tools/x64/postgresql/postgresql-17/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-17/bin/pg_restore
index c7c75c34..2ee091ca 100755
Binary files a/assets/tools/x64/postgresql/postgresql-17/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-17/bin/pg_restore differ
diff --git a/assets/tools/x64/postgresql/postgresql-17/bin/psql b/assets/tools/x64/postgresql/postgresql-17/bin/psql
index 01b099e9..6ed35f41 100755
Binary files a/assets/tools/x64/postgresql/postgresql-17/bin/psql and b/assets/tools/x64/postgresql/postgresql-17/bin/psql differ
diff --git a/assets/tools/x64/postgresql/postgresql-18/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-18/bin/pg_dump
index 0fa75a3c..54170049 100755
Binary files a/assets/tools/x64/postgresql/postgresql-18/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-18/bin/pg_dump differ
diff --git a/assets/tools/x64/postgresql/postgresql-18/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-18/bin/pg_restore
index 6e041e78..fcd66b01 100755
Binary files a/assets/tools/x64/postgresql/postgresql-18/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-18/bin/pg_restore differ
diff --git a/assets/tools/x64/postgresql/postgresql-18/bin/psql b/assets/tools/x64/postgresql/postgresql-18/bin/psql
index 8460c000..c5cad676 100755
Binary files a/assets/tools/x64/postgresql/postgresql-18/bin/psql and b/assets/tools/x64/postgresql/postgresql-18/bin/psql differ
diff --git a/backend/internal/features/tests/logical/postgresql/backup_restore_test.go b/backend/internal/features/tests/logical/postgresql/backup_restore_test.go
index d27339cd..6c9427b7 100644
--- a/backend/internal/features/tests/logical/postgresql/backup_restore_test.go
+++ b/backend/internal/features/tests/logical/postgresql/backup_restore_test.go
@@ -175,6 +175,10 @@ func Test_PostgresqlBackupRestore_AcrossSupportedVersions(t *testing.T) {
testBackupRestoreWithReadOnlyUserForVersion(t, endpoint, dbVersion.tag)
})
+ t.Run("Test_BackupPostgresql_WhenUserCannotReadSequences_BackupFails", func(t *testing.T) {
+ testBackupFailsWhenUserCannotReadSequencesForVersion(t, endpoint, dbVersion.tag)
+ })
+
t.Run("Test_BackupAndRestorePostgresql_WithSkipUserMappings_RestoreIsSuccessful", func(t *testing.T) {
testBackupRestoreSkipUserMappingsForVersion(t, endpoint, dbVersion.tag)
})
@@ -1163,6 +1167,105 @@ func testBackupRestoreWithReadOnlyUserForVersion(t *testing.T, endpoint containe
workspaces_testing.RemoveTestWorkspace(workspace, router)
}
+// pg_dump 18.0 and 18.1 emitted wrong sequence values instead of failing when the
+// dumping role could not read a sequence, so a backup completed while silently
+// losing every sequence position (issue #725, fixed upstream in PostgreSQL 18.2).
+func testBackupFailsWhenUserCannotReadSequencesForVersion(
+ t *testing.T,
+ endpoint containers.Endpoint,
+ pgVersion string,
+) {
+ container, err := connectToPostgresEndpoint(t, endpoint)
+ assert.NoError(t, err)
+ defer func() {
+ if container.DB != nil {
+ container.DB.Close()
+ }
+ }()
+
+ uniqueSuffix := uuid.New().String()[:8]
+ tableName := fmt.Sprintf("test_data_%s", uniqueSuffix)
+ sequenceName := tableName + "_id_seq"
+ restrictedUsername := fmt.Sprintf("no_sequence_reader_%s", uniqueSuffix)
+ restrictedPassword := uuid.New().String()
+
+ _, err = container.DB.Exec(createAndFillTableQuery(tableName))
+ assert.NoError(t, err)
+
+ _, err = container.DB.Exec(fmt.Sprintf(`
+ CREATE ROLE %[1]s LOGIN PASSWORD '%[2]s';
+ GRANT CONNECT ON DATABASE %[3]s TO %[1]s;
+ GRANT USAGE ON SCHEMA public TO %[1]s;
+ GRANT SELECT ON TABLE %[4]s TO %[1]s;
+ REVOKE ALL ON SEQUENCE %[5]s FROM PUBLIC, %[1]s;
+ `, restrictedUsername, restrictedPassword, container.Database, tableName, sequenceName))
+ assert.NoError(t, err)
+
+ defer func() {
+ _, _ = container.DB.Exec(fmt.Sprintf(`
+ DROP TABLE IF EXISTS %s;
+ REASSIGN OWNED BY %s TO %s;
+ DROP OWNED BY %s;
+ DROP ROLE IF EXISTS %s;
+ `, tableName, restrictedUsername, container.Username, restrictedUsername, restrictedUsername))
+ }()
+
+ router := logicaltesting.CreateTestRouter()
+ user := users_testing.CreateTestUser(users_enums.UserRoleMember)
+ workspace := workspaces_testing.CreateTestWorkspace("Sequence Privileges Workspace", user, router)
+
+ storage := storages.CreateTestStorage(workspace.ID)
+
+ database := createDatabaseViaAPI(
+ t, router, "Sequence Privileges Database", workspace.ID,
+ container.Host, container.Port,
+ container.Username, container.Password, container.Database,
+ user.Token,
+ )
+
+ restrictedDatabase := updateDatabaseCredentialsViaAPI(
+ t, router, database,
+ restrictedUsername, restrictedPassword,
+ user.Token,
+ )
+
+ logicaltesting.EnableBackupsViaAPI(
+ t, router, restrictedDatabase.ID, storage.ID,
+ backups_core_enums.BackupEncryptionNone, user.Token,
+ )
+
+ logicaltesting.CreateBackupViaAPI(t, router, restrictedDatabase.ID, user.Token)
+
+ backup := logicaltesting.WaitForBackupTerminalStatus(
+ t, router, restrictedDatabase.ID, user.Token, 5*time.Minute,
+ )
+
+ assert.Equal(
+ t,
+ backups_core_logical.BackupStatusFailed,
+ backup.Status,
+ "PostgreSQL %s: backup must fail when the dumping role cannot read sequences", pgVersion,
+ )
+
+ // PG 18 says "failed to get data for sequence ..." where earlier majors say
+ // "permission denied for sequence ..."; both name the sequence they choked on.
+ if assert.NotNil(t, backup.FailMessage) {
+ assert.Contains(t, *backup.FailMessage, sequenceName)
+ }
+
+ _ = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
+
+ test_utils.MakeDeleteRequest(
+ t,
+ router,
+ "/api/v1/databases/"+restrictedDatabase.ID.String(),
+ "Bearer "+user.Token,
+ http.StatusNoContent,
+ )
+ storages.RemoveTestStorage(storage.ID)
+ workspaces_testing.RemoveTestWorkspace(workspace, router)
+}
+
func testSchemaSelectionOnlySpecifiedSchemasForVersion(
t *testing.T,
endpoint containers.Endpoint,
diff --git a/backend/internal/features/tests/logical/shared/backups.go b/backend/internal/features/tests/logical/shared/backups.go
index e11af34b..7f25b489 100644
--- a/backend/internal/features/tests/logical/shared/backups.go
+++ b/backend/internal/features/tests/logical/shared/backups.go
@@ -73,8 +73,6 @@ func CreateBackupViaAPI(
)
}
-// WaitForBackupCompletion polls the backups API until the latest backup for the
-// database reaches Completed, failing the test on Failed or timeout.
func WaitForBackupCompletion(
t *testing.T,
router *gin.Engine,
@@ -82,45 +80,20 @@ func WaitForBackupCompletion(
token string,
timeout time.Duration,
) *backups_core_logical.LogicalBackup {
- startTime := time.Now()
- pollInterval := 500 * time.Millisecond
-
- for {
- if time.Since(startTime) > timeout {
- t.Fatalf("Timeout waiting for backup completion after %v", timeout)
- }
-
- var response backups_dto_logical.GetBackupsResponse
- test_utils.MakeGetRequestAndUnmarshal(
- t,
- router,
- fmt.Sprintf("/api/v1/backups?database_id=%s&limit=1", databaseID.String()),
- "Bearer "+token,
- http.StatusOK,
- &response,
- )
+ backup := WaitForBackupTerminalStatus(t, router, databaseID, token, timeout)
- if len(response.Backups) > 0 {
- backup := response.Backups[0]
- if backup.Status == backups_core_logical.BackupStatusCompleted {
- return backup
- }
- if backup.Status == backups_core_logical.BackupStatusFailed {
- failMsg := "unknown error"
- if backup.FailMessage != nil {
- failMsg = *backup.FailMessage
- }
- t.Fatalf("Backup failed: %s", failMsg)
- }
+ if backup.Status != backups_core_logical.BackupStatusCompleted {
+ failMessage := "unknown error"
+ if backup.FailMessage != nil {
+ failMessage = *backup.FailMessage
}
- time.Sleep(pollInterval)
+ t.Fatalf("backup reached %s instead of completing: %s", backup.Status, failMessage)
}
+
+ return backup
}
-// WaitForBackupTerminalStatus polls until the latest backup reaches any terminal
-// status (Completed, Failed, Canceled) and returns it, failing on timeout. Used
-// by the issue-582 regression which asserts a failed backup does not hang.
func WaitForBackupTerminalStatus(
t *testing.T,
router *gin.Engine,
@@ -143,11 +116,11 @@ func WaitForBackupTerminalStatus(
)
if len(response.Backups) > 0 {
- b := response.Backups[0]
- if b.Status == backups_core_logical.BackupStatusCompleted ||
- b.Status == backups_core_logical.BackupStatusFailed ||
- b.Status == backups_core_logical.BackupStatusCanceled {
- return b
+ newestBackup := response.Backups[0]
+ if newestBackup.Status == backups_core_logical.BackupStatusCompleted ||
+ newestBackup.Status == backups_core_logical.BackupStatusFailed ||
+ newestBackup.Status == backups_core_logical.BackupStatusCanceled {
+ return newestBackup
}
}
@@ -155,8 +128,7 @@ func WaitForBackupTerminalStatus(
}
t.Fatalf(
- "backup for database %s did not reach a terminal status within %v "+
- "(issue #582: backup hangs forever when SaveFile fails)",
+ "backup for database %s did not reach a terminal status within %v",
databaseID,
timeout,
)