PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (postgresql): Update bundled pg_dump past the 18.1 sequence-value bug (issue #725)
Rostislav Duginddb631ef1afc
diff --git a/assets/tools/README.md b/assets/tools/README.mdindex 3daf0a72..9ae491f3 100644--- a/assets/tools/README.md+++ b/assets/tools/README.md@@ -24,6 +24,23 @@ assets/tools/<arch>/| `linux` / `amd64` | `x64` | ~160 MB || `linux` / `arm64` | `arm` | ~125 MB |+Bundled PostgreSQL minors (identical on both arches, Debian bookworm pgdg):++| major | minor | source package |+|-------|-------|-----------------------------------|+| 12 | 12.22 | `12.22-3.pgdg12+1` (final, EOL) |+| 13 | 13.23 | `13.23-1.pgdg12+1` (final, EOL) |+| 14 | 14.23 | `14.23-1.pgdg12+1` |+| 15 | 15.18 | `15.18-1.pgdg12+1` |+| 16 | 16.14 | `16.14-1.pgdg12+1` |+| 17 | 17.10 | `17.10-1.pgdg12+1` |+| 18 | 18.4 | `18.4-1.pgdg12+1` |++Majors 12 and 13 are past upstream EOL, so those minors are the last ones that+will ever exist. Keep every binary within one major on the same package build —+a `pg_dump` older than its sibling `pg_basebackup` is how issue #725 (pg_dump+18.1 silently emitting wrong sequence values) survived unnoticed.+Notes:- MySQL `5.7` is amd64-only — `arm/mysql/mysql-5.7/` is intentionally absent.- MariaDB ships two client versions: legacy `10.6` (for MariaDB servers 5.5 / 10.1) and modern `12.1` (for 10.2+). The mapping lives in `tools/mariadb.go`.@@ -31,4 +48,7 @@ Notes:To refresh a tool set, drop the corresponding `bin/` contents in placeand commit. There are no install scripts in this directory; binaries-are sourced from the upstream vendor downloads.+are sourced from the upstream vendor downloads. For PostgreSQL that means+`postgresql-client-<major>` from `apt.postgresql.org` for `bookworm`, both+`amd64` and `arm64`, matching the `debian:bookworm-slim` runtime base in the+Dockerfile. Update the minor table above in the same commit.diff --git a/assets/tools/arm/postgresql/postgresql-14/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-14/bin/pg_dumpindex 7c6bcade..9b7d7b6c 100755Binary files a/assets/tools/arm/postgresql/postgresql-14/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-14/bin/pg_dump differdiff --git a/assets/tools/arm/postgresql/postgresql-14/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-14/bin/pg_restoreindex 6df17ae2..3f3df196 100755Binary files a/assets/tools/arm/postgresql/postgresql-14/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-14/bin/pg_restore differdiff --git a/assets/tools/arm/postgresql/postgresql-14/bin/psql b/assets/tools/arm/postgresql/postgresql-14/bin/psqlindex 8616815a..6e999006 100755Binary files a/assets/tools/arm/postgresql/postgresql-14/bin/psql and b/assets/tools/arm/postgresql/postgresql-14/bin/psql differdiff --git a/assets/tools/arm/postgresql/postgresql-15/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-15/bin/pg_dumpindex 8eecace9..4ba39734 100755Binary files a/assets/tools/arm/postgresql/postgresql-15/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-15/bin/pg_dump differdiff --git a/assets/tools/arm/postgresql/postgresql-15/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-15/bin/pg_restoreindex 6ad5e4be..1cfa8839 100755Binary files a/assets/tools/arm/postgresql/postgresql-15/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-15/bin/pg_restore differdiff --git a/assets/tools/arm/postgresql/postgresql-15/bin/psql b/assets/tools/arm/postgresql/postgresql-15/bin/psqlindex ce119392..ba0f0ecd 100755Binary files a/assets/tools/arm/postgresql/postgresql-15/bin/psql and b/assets/tools/arm/postgresql/postgresql-15/bin/psql differdiff --git a/assets/tools/arm/postgresql/postgresql-16/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-16/bin/pg_dumpindex 88ebd978..6ca72ef3 100755Binary files a/assets/tools/arm/postgresql/postgresql-16/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-16/bin/pg_dump differdiff --git a/assets/tools/arm/postgresql/postgresql-16/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-16/bin/pg_restoreindex eee24bb1..c7a43811 100755Binary files a/assets/tools/arm/postgresql/postgresql-16/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-16/bin/pg_restore differdiff --git a/assets/tools/arm/postgresql/postgresql-16/bin/psql b/assets/tools/arm/postgresql/postgresql-16/bin/psqlindex e96c287b..019c475a 100755Binary files a/assets/tools/arm/postgresql/postgresql-16/bin/psql and b/assets/tools/arm/postgresql/postgresql-16/bin/psql differdiff --git a/assets/tools/arm/postgresql/postgresql-17/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-17/bin/pg_dumpindex 2eb8f0eb..1db9e773 100755Binary files a/assets/tools/arm/postgresql/postgresql-17/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-17/bin/pg_dump differdiff --git a/assets/tools/arm/postgresql/postgresql-17/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-17/bin/pg_restoreindex 87a71cc6..e1f0a0a2 100755Binary files a/assets/tools/arm/postgresql/postgresql-17/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-17/bin/pg_restore differdiff --git a/assets/tools/arm/postgresql/postgresql-17/bin/psql b/assets/tools/arm/postgresql/postgresql-17/bin/psqlindex d981a9ce..0e078476 100755Binary files a/assets/tools/arm/postgresql/postgresql-17/bin/psql and b/assets/tools/arm/postgresql/postgresql-17/bin/psql differdiff --git a/assets/tools/arm/postgresql/postgresql-18/bin/pg_dump b/assets/tools/arm/postgresql/postgresql-18/bin/pg_dumpindex e770af77..f1554f6f 100755Binary files a/assets/tools/arm/postgresql/postgresql-18/bin/pg_dump and b/assets/tools/arm/postgresql/postgresql-18/bin/pg_dump differdiff --git a/assets/tools/arm/postgresql/postgresql-18/bin/pg_restore b/assets/tools/arm/postgresql/postgresql-18/bin/pg_restoreindex d4a58fe1..d0609070 100755Binary files a/assets/tools/arm/postgresql/postgresql-18/bin/pg_restore and b/assets/tools/arm/postgresql/postgresql-18/bin/pg_restore differdiff --git a/assets/tools/arm/postgresql/postgresql-18/bin/psql b/assets/tools/arm/postgresql/postgresql-18/bin/psqlindex db4c8e27..17bff46f 100755Binary files a/assets/tools/arm/postgresql/postgresql-18/bin/psql and b/assets/tools/arm/postgresql/postgresql-18/bin/psql differdiff --git a/assets/tools/x64/postgresql/postgresql-14/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-14/bin/pg_dumpindex 955b2435..ff1bc4ce 100755Binary files a/assets/tools/x64/postgresql/postgresql-14/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-14/bin/pg_dump differdiff --git a/assets/tools/x64/postgresql/postgresql-14/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-14/bin/pg_restoreindex 355c3159..272c5fa4 100755Binary files a/assets/tools/x64/postgresql/postgresql-14/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-14/bin/pg_restore differdiff --git a/assets/tools/x64/postgresql/postgresql-14/bin/psql b/assets/tools/x64/postgresql/postgresql-14/bin/psqlindex 48192354..8c89979a 100755Binary files a/assets/tools/x64/postgresql/postgresql-14/bin/psql and b/assets/tools/x64/postgresql/postgresql-14/bin/psql differdiff --git a/assets/tools/x64/postgresql/postgresql-15/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-15/bin/pg_dumpindex 8881dabb..7b1c271d 100755Binary files a/assets/tools/x64/postgresql/postgresql-15/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-15/bin/pg_dump differdiff --git a/assets/tools/x64/postgresql/postgresql-15/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-15/bin/pg_restoreindex cbd44ba7..a8546a51 100755Binary files a/assets/tools/x64/postgresql/postgresql-15/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-15/bin/pg_restore differdiff --git a/assets/tools/x64/postgresql/postgresql-15/bin/psql b/assets/tools/x64/postgresql/postgresql-15/bin/psqlindex 260408d6..e3bb8add 100755Binary files a/assets/tools/x64/postgresql/postgresql-15/bin/psql and b/assets/tools/x64/postgresql/postgresql-15/bin/psql differdiff --git a/assets/tools/x64/postgresql/postgresql-16/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-16/bin/pg_dumpindex 8d20464a..34e81ed9 100755Binary files a/assets/tools/x64/postgresql/postgresql-16/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-16/bin/pg_dump differdiff --git a/assets/tools/x64/postgresql/postgresql-16/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-16/bin/pg_restoreindex 929e15aa..fa073ec1 100755Binary files a/assets/tools/x64/postgresql/postgresql-16/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-16/bin/pg_restore differdiff --git a/assets/tools/x64/postgresql/postgresql-16/bin/psql b/assets/tools/x64/postgresql/postgresql-16/bin/psqlindex 3855edd9..e4982ebb 100755Binary files a/assets/tools/x64/postgresql/postgresql-16/bin/psql and b/assets/tools/x64/postgresql/postgresql-16/bin/psql differdiff --git a/assets/tools/x64/postgresql/postgresql-17/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-17/bin/pg_dumpindex 51a235d7..b650f887 100755Binary files a/assets/tools/x64/postgresql/postgresql-17/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-17/bin/pg_dump differdiff --git a/assets/tools/x64/postgresql/postgresql-17/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-17/bin/pg_restoreindex c7c75c34..2ee091ca 100755Binary files a/assets/tools/x64/postgresql/postgresql-17/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-17/bin/pg_restore differdiff --git a/assets/tools/x64/postgresql/postgresql-17/bin/psql b/assets/tools/x64/postgresql/postgresql-17/bin/psqlindex 01b099e9..6ed35f41 100755Binary files a/assets/tools/x64/postgresql/postgresql-17/bin/psql and b/assets/tools/x64/postgresql/postgresql-17/bin/psql differdiff --git a/assets/tools/x64/postgresql/postgresql-18/bin/pg_dump b/assets/tools/x64/postgresql/postgresql-18/bin/pg_dumpindex 0fa75a3c..54170049 100755Binary files a/assets/tools/x64/postgresql/postgresql-18/bin/pg_dump and b/assets/tools/x64/postgresql/postgresql-18/bin/pg_dump differdiff --git a/assets/tools/x64/postgresql/postgresql-18/bin/pg_restore b/assets/tools/x64/postgresql/postgresql-18/bin/pg_restoreindex 6e041e78..fcd66b01 100755Binary files a/assets/tools/x64/postgresql/postgresql-18/bin/pg_restore and b/assets/tools/x64/postgresql/postgresql-18/bin/pg_restore differdiff --git a/assets/tools/x64/postgresql/postgresql-18/bin/psql b/assets/tools/x64/postgresql/postgresql-18/bin/psqlindex 8460c000..c5cad676 100755Binary files a/assets/tools/x64/postgresql/postgresql-18/bin/psql and b/assets/tools/x64/postgresql/postgresql-18/bin/psql differdiff --git a/backend/internal/features/tests/logical/postgresql/backup_restore_test.go b/backend/internal/features/tests/logical/postgresql/backup_restore_test.goindex d27339cd..6c9427b7 100644--- a/backend/internal/features/tests/logical/postgresql/backup_restore_test.go+++ b/backend/internal/features/tests/logical/postgresql/backup_restore_test.go@@ -175,6 +175,10 @@ func Test_PostgresqlBackupRestore_AcrossSupportedVersions(t *testing.T) {testBackupRestoreWithReadOnlyUserForVersion(t, endpoint, dbVersion.tag)})+ t.Run("Test_BackupPostgresql_WhenUserCannotReadSequences_BackupFails", func(t *testing.T) {+ testBackupFailsWhenUserCannotReadSequencesForVersion(t, endpoint, dbVersion.tag)+ })+t.Run("Test_BackupAndRestorePostgresql_WithSkipUserMappings_RestoreIsSuccessful", func(t *testing.T) {testBackupRestoreSkipUserMappingsForVersion(t, endpoint, dbVersion.tag)})@@ -1163,6 +1167,105 @@ func testBackupRestoreWithReadOnlyUserForVersion(t *testing.T, endpoint containeworkspaces_testing.RemoveTestWorkspace(workspace, router)}+// pg_dump 18.0 and 18.1 emitted wrong sequence values instead of failing when the+// dumping role could not read a sequence, so a backup completed while silently+// losing every sequence position (issue #725, fixed upstream in PostgreSQL 18.2).+func testBackupFailsWhenUserCannotReadSequencesForVersion(+ t *testing.T,+ endpoint containers.Endpoint,+ pgVersion string,+) {+ container, err := connectToPostgresEndpoint(t, endpoint)+ assert.NoError(t, err)+ defer func() {+ if container.DB != nil {+ container.DB.Close()+ }+ }()++ uniqueSuffix := uuid.New().String()[:8]+ tableName := fmt.Sprintf("test_data_%s", uniqueSuffix)+ sequenceName := tableName + "_id_seq"+ restrictedUsername := fmt.Sprintf("no_sequence_reader_%s", uniqueSuffix)+ restrictedPassword := uuid.New().String()++ _, err = container.DB.Exec(createAndFillTableQuery(tableName))+ assert.NoError(t, err)++ _, err = container.DB.Exec(fmt.Sprintf(`+ CREATE ROLE %[1]s LOGIN PASSWORD '%[2]s';+ GRANT CONNECT ON DATABASE %[3]s TO %[1]s;+ GRANT USAGE ON SCHEMA public TO %[1]s;+ GRANT SELECT ON TABLE %[4]s TO %[1]s;+ REVOKE ALL ON SEQUENCE %[5]s FROM PUBLIC, %[1]s;+ `, restrictedUsername, restrictedPassword, container.Database, tableName, sequenceName))+ assert.NoError(t, err)++ defer func() {+ _, _ = container.DB.Exec(fmt.Sprintf(`+ DROP TABLE IF EXISTS %s;+ REASSIGN OWNED BY %s TO %s;+ DROP OWNED BY %s;+ DROP ROLE IF EXISTS %s;+ `, tableName, restrictedUsername, container.Username, restrictedUsername, restrictedUsername))+ }()++ router := logicaltesting.CreateTestRouter()+ user := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Sequence Privileges Workspace", user, router)++ storage := storages.CreateTestStorage(workspace.ID)++ database := createDatabaseViaAPI(+ t, router, "Sequence Privileges Database", workspace.ID,+ container.Host, container.Port,+ container.Username, container.Password, container.Database,+ user.Token,+ )++ restrictedDatabase := updateDatabaseCredentialsViaAPI(+ t, router, database,+ restrictedUsername, restrictedPassword,+ user.Token,+ )++ logicaltesting.EnableBackupsViaAPI(+ t, router, restrictedDatabase.ID, storage.ID,+ backups_core_enums.BackupEncryptionNone, user.Token,+ )++ logicaltesting.CreateBackupViaAPI(t, router, restrictedDatabase.ID, user.Token)++ backup := logicaltesting.WaitForBackupTerminalStatus(+ t, router, restrictedDatabase.ID, user.Token, 5*time.Minute,+ )++ assert.Equal(+ t,+ backups_core_logical.BackupStatusFailed,+ backup.Status,+ "PostgreSQL %s: backup must fail when the dumping role cannot read sequences", pgVersion,+ )++ // PG 18 says "failed to get data for sequence ..." where earlier majors say+ // "permission denied for sequence ..."; both name the sequence they choked on.+ if assert.NotNil(t, backup.FailMessage) {+ assert.Contains(t, *backup.FailMessage, sequenceName)+ }++ _ = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))++ test_utils.MakeDeleteRequest(+ t,+ router,+ "/api/v1/databases/"+restrictedDatabase.ID.String(),+ "Bearer "+user.Token,+ http.StatusNoContent,+ )+ storages.RemoveTestStorage(storage.ID)+ workspaces_testing.RemoveTestWorkspace(workspace, router)+}+func testSchemaSelectionOnlySpecifiedSchemasForVersion(t *testing.T,endpoint containers.Endpoint,diff --git a/backend/internal/features/tests/logical/shared/backups.go b/backend/internal/features/tests/logical/shared/backups.goindex e11af34b..7f25b489 100644--- a/backend/internal/features/tests/logical/shared/backups.go+++ b/backend/internal/features/tests/logical/shared/backups.go@@ -73,8 +73,6 @@ func CreateBackupViaAPI()}-// WaitForBackupCompletion polls the backups API until the latest backup for the-// database reaches Completed, failing the test on Failed or timeout.func WaitForBackupCompletion(t *testing.T,router *gin.Engine,@@ -82,45 +80,20 @@ func WaitForBackupCompletion(token string,timeout time.Duration,) *backups_core_logical.LogicalBackup {- startTime := time.Now()- pollInterval := 500 * time.Millisecond-- for {- if time.Since(startTime) > timeout {- t.Fatalf("Timeout waiting for backup completion after %v", timeout)- }-- var response backups_dto_logical.GetBackupsResponse- test_utils.MakeGetRequestAndUnmarshal(- t,- router,- fmt.Sprintf("/api/v1/backups?database_id=%s&limit=1", databaseID.String()),- "Bearer "+token,- http.StatusOK,- &response,- )+ backup := WaitForBackupTerminalStatus(t, router, databaseID, token, timeout)- if len(response.Backups) > 0 {- backup := response.Backups[0]- if backup.Status == backups_core_logical.BackupStatusCompleted {- return backup- }- if backup.Status == backups_core_logical.BackupStatusFailed {- failMsg := "unknown error"- if backup.FailMessage != nil {- failMsg = *backup.FailMessage- }- t.Fatalf("Backup failed: %s", failMsg)- }+ if backup.Status != backups_core_logical.BackupStatusCompleted {+ failMessage := "unknown error"+ if backup.FailMessage != nil {+ failMessage = *backup.FailMessage}- time.Sleep(pollInterval)+ t.Fatalf("backup reached %s instead of completing: %s", backup.Status, failMessage)}++ return backup}-// WaitForBackupTerminalStatus polls until the latest backup reaches any terminal-// status (Completed, Failed, Canceled) and returns it, failing on timeout. Used-// by the issue-582 regression which asserts a failed backup does not hang.func WaitForBackupTerminalStatus(t *testing.T,router *gin.Engine,@@ -143,11 +116,11 @@ func WaitForBackupTerminalStatus()if len(response.Backups) > 0 {- b := response.Backups[0]- if b.Status == backups_core_logical.BackupStatusCompleted ||- b.Status == backups_core_logical.BackupStatusFailed ||- b.Status == backups_core_logical.BackupStatusCanceled {- return b+ newestBackup := response.Backups[0]+ if newestBackup.Status == backups_core_logical.BackupStatusCompleted ||+ newestBackup.Status == backups_core_logical.BackupStatusFailed ||+ newestBackup.Status == backups_core_logical.BackupStatusCanceled {+ return newestBackup}}@@ -155,8 +128,7 @@ func WaitForBackupTerminalStatus(}t.Fatalf(- "backup for database %s did not reach a terminal status within %v "+- "(issue #582: backup hangs forever when SaveFile fails)",+ "backup for database %s did not reach a terminal status within %v",databaseID,timeout,)