PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (restore): Explain what to do if confings are present not on default PG path
Rostislav Dugin8d38c8f2a6a1
diff --git a/backend/internal/features/backups/backups/controllers/physical/controller_test.go b/backend/internal/features/backups/backups/controllers/physical/controller_test.goindex a5fabc89..8af9f3f3 100644--- a/backend/internal/features/backups/backups/controllers/physical/controller_test.go+++ b/backend/internal/features/backups/backups/controllers/physical/controller_test.go@@ -946,6 +946,31 @@ func Test_GetRecoveryScript_ReturnsRunnableShellScript(t *testing.T) {"WAL is decompressed on the host up front; the script must not wire zstd into restore_command")}+// A cluster whose source keeps its configuration outside the data directory (the Debian/Ubuntu+// layout) restores without postgresql.conf and refuses to start. The served script must diagnose+// that and instruct - never invent an access-control policy for the restored data (issue #685).+func Test_GetRecoveryScript_IncludesConfigFileCheck(t *testing.T) {+ router := newPhysicalControllerRouter()++ response := test_utils.MakeGetRequest(t, router,+ "/api/v1/backups/physical/recovery-script", "", http.StatusOK)++ servedScript := string(response.Body)++ assert.Contains(t, servedScript, "ACTION REQUIRED: the restored cluster has no configuration files",+ "the script must name the failure instead of leaving it to the server's startup error")+ assert.Contains(t, servedScript, "/etc/postgresql/",+ "the instructions must name where Debian/Ubuntu keeps the originals")+ assert.Contains(t, servedScript, "SHOW config_file",+ "the instructions must show how to ask the source cluster for the real paths")+ assert.Contains(t, servedScript, "pg_lsclusters",+ "the instructions must show how to tell two clusters on one host apart")+ // Blanket, not rule-shaped: any future mention deserves a second look, because the whole point+ // is that the restore never hands the user an unauthenticated rule to paste.+ assert.NotContains(t, servedScript, "trust",+ "the script must never emit a trust rule for the restored cluster")+}+func Test_GetRestoreStream_TokenIsSingleUse(t *testing.T) {prereqs := createPhysicalControllerPrereqs(t)diff --git a/backend/internal/features/backups/backups/controllers/physical/recovery_script.sh b/backend/internal/features/backups/backups/controllers/physical/recovery_script.shindex b35e63b1..1c8e9e48 100644--- a/backend/internal/features/backups/backups/controllers/physical/recovery_script.sh+++ b/backend/internal/features/backups/backups/controllers/physical/recovery_script.sh@@ -180,6 +180,166 @@ control_setting() {echo "$2" | sed -n "s/^$1 setting: *//p" | tr -d '[:space:]'}+# Major of the server inside --combine-image, so a PG 18 backup handed a postgres:17 image is caught+# before the user boots it. The host path gets this from pg_ctl --version; the container needs a run.+image_server_major() {+ pg_major_from_banner "$(docker run --rm "$COMBINE_IMAGE" postgres --version 2>/dev/null || true)"+}++# A base backup copies only the data directory. Debian/Ubuntu keep postgresql.conf, pg_hba.conf and+# pg_ident.conf in /etc/postgresql/<major>/<cluster> - outside it - so a cluster restored from such+# a source has no configuration and no server will start it. The general rule is any cluster whose+# config_file points outside data_directory; RPM, Arch, Alpine and the official image are unaffected.+list_missing_cluster_config() {+ for config in postgresql.conf pg_hba.conf pg_ident.conf; do+ if [ ! -f "$DATA_DIR/$config" ]; then+ echo "$config"+ fi+ done+}++is_cluster_dir_complete() {+ for marker in PG_VERSION base global pg_wal; do+ if [ ! -e "$DATA_DIR/$marker" ]; then+ return 1+ fi+ done++ return 0+}++# Only the restored major is searched: an /etc/postgresql/16 hit next to a PG 17 backup is a+# different cluster and must not be offered. The cluster name is conventionally "main" but+# pg_createcluster allows any, so it is globbed rather than assumed.+list_debian_config_dirs() {+ for candidate in "/etc/postgresql/$RESTORED_MAJOR"/*/; do+ if [ -f "$candidate/postgresql.conf" ]; then+ echo "${candidate%/}"+ fi+ done+}++print_config_file_readability() {+ for config in postgresql.conf pg_hba.conf pg_ident.conf; do+ if [ ! -e "$1/$config" ]; then+ echo " $config - not there"+ elif [ -r "$1/$config" ]; then+ echo " $config - readable"+ else+ echo " $config - NOT readable as '$(id -un)', copy it with sudo"+ fi+ done+}++# sudo is needed per directory, not globally: Debian ships pg_hba.conf tighter than postgresql.conf,+# and two clusters on one host can differ in ownership.+config_copy_prefix() {+ for config in postgresql.conf pg_hba.conf pg_ident.conf; do+ if [ -e "$1/$config" ] && [ ! -r "$1/$config" ]; then+ echo "sudo cp"+ return 0+ fi+ done++ echo "cp"+}++print_missing_config_instructions() {+ missing_cluster_config="$1"+ config_dirs="$(list_debian_config_dirs)"++ echo+ echo "================================================================================"+ echo "ACTION REQUIRED: the restored cluster has no configuration files"+ echo "================================================================================"+ echo " Missing in $DATA_DIR:"+ for config in $missing_cluster_config; do+ echo " $config"+ done+ echo+ echo " Do all of this BEFORE the chown in step 1 above: while the directory is still yours the"+ echo " recursive chown then picks these files up too, and no extra ownership fixing is needed."+ echo+ if is_cluster_dir_complete; then+ echo " The data itself is complete - PG_VERSION, base, global and pg_wal are all there."+ echo " This is NOT a wrong-directory problem: PostgreSQL on Debian/Ubuntu keeps its"+ echo " configuration in /etc/postgresql/<major>/<cluster>, outside the data directory, and a"+ echo " base backup copies only the data directory. Those files are not in the backup."+ else+ echo " warning: '$DATA_DIR' is also missing parts of a cluster (PG_VERSION, base, global or"+ echo " pg_wal). Treat the reconstruction itself as suspect before going further."+ fi+ echo+ echo " 1. Ask the SOURCE cluster where its configuration lives (works on any distribution;"+ echo " needs superuser or pg_read_all_settings):"+ echo " psql -h <source-host> -U postgres -Atc \"SHOW config_file\""+ echo " psql -h <source-host> -U postgres -Atc \"SHOW hba_file\""+ echo " psql -h <source-host> -U postgres -Atc \"SHOW ident_file\""+ echo++ if [ -z "$config_dirs" ]; then+ echo " 2. No PostgreSQL $RESTORED_MAJOR configuration was found under /etc/postgresql on this"+ echo " host, so this is not the source machine. Copy the three files from the source to"+ echo " '$DATA_DIR' - on Debian/Ubuntu they are in /etc/postgresql/$RESTORED_MAJOR/main."+ else+ echo " 2. Found a PostgreSQL $RESTORED_MAJOR configuration on this host:"+ for config_dir in $config_dirs; do+ echo " $config_dir"+ print_config_file_readability "$config_dir"+ data_directory="$(sed -n "s/^[[:space:]]*data_directory[[:space:]]*=[[:space:]]*//p" \+ "$config_dir/postgresql.conf" 2>/dev/null | head -n1)"+ if [ -n "$data_directory" ]; then+ echo " serves data_directory $data_directory"+ fi+ done+ echo " (pg_lsclusters lists every cluster on a Debian/Ubuntu host.)"+ echo " Copy them in - pick the cluster this backup came from:"+ for config_dir in $config_dirs; do+ echo " $(config_copy_prefix "$config_dir") '$config_dir'/postgresql.conf '$config_dir'/pg_hba.conf \\"+ echo " '$config_dir'/pg_ident.conf '$DATA_DIR'/"+ done+ fi++ echo+ echo " 3. Then edit '$DATA_DIR/postgresql.conf'. These point at paths that do not exist here and"+ echo " will stop the server:"+ echo " data_directory, hba_file, ident_file, external_pid_file -> comment out"+ echo " ssl, ssl_cert_file, ssl_key_file, ssl_ca_file -> comment out, or copy the certs"+ echo " ('ssl = on' alone fails:"+ echo " it looks for server.crt here)"+ echo " include_dir = 'conf.d' -> mkdir '$DATA_DIR/conf.d'"+ echo " In a container also set listen_addresses = '*' - Debian defaults it to localhost,"+ echo " which leaves the published port dead."+ echo+ echo " If the source host is gone and the originals are unobtainable, this is the minimum that"+ echo " starts a cluster. Databasus does not write it for you - the access rules are yours to"+ echo " decide, so review them first:"+ # No backslash escapes in anything printed here: dash - /bin/sh on the very distributions this+ # block is for - expands them in echo, so a printf '\n' would arrive split across lines.++ # A container needs listen_addresses or its published port stays dead; a host start is left on+ # PostgreSQL's own localhost default rather than widening exposure on the user's behalf.+ if [ -n "$COMBINE_IMAGE" ]; then+ echo " echo \"listen_addresses = '*'\" > '$DATA_DIR/postgresql.conf'"+ else+ echo " : > '$DATA_DIR/postgresql.conf'"+ fi+ echo " { echo 'local all all peer'; echo 'host all all all scram-sha-256'; } > '$DATA_DIR/pg_hba.conf'"+ echo " : > '$DATA_DIR/pg_ident.conf'"+ echo " chmod 600 '$DATA_DIR'/*.conf"+ echo+ echo " Not the same thing as a wrong directory. If you later hit this error while the three"+ echo " files ARE in '$DATA_DIR', then what you mounted is not what the server uses as PGDATA."+ echo " PostgreSQL $RESTORED_MAJOR in the official image expects:"+ if [ "${RESTORED_MAJOR:-0}" -ge 18 ] 2>/dev/null; then+ echo " -v '$OUT_ABS:/var/lib/postgresql' (PGDATA is nested at $CONTAINER_PGDATA)"+ echo " or bind '$DATA_DIR' straight at '$CONTAINER_PGDATA'"+ else+ echo " -v '$DATA_DIR:$CONTAINER_PGDATA'"+ fi+ echo "================================================================================"+}+# Resolve the PostgreSQL tools off --pg-bin when they are not on PATH; every later# pg_combinebackup / pg_ctl lookup then finds them.if [ -n "$PG_BIN" ]; then@@ -290,6 +450,26 @@ doneRESTORED_MAJOR="$(tr -d '[:space:]' <"$RECON_DIR/full/PG_VERSION" 2>/dev/null || true)"DATA_DIR="$(host_cluster_dir "$OUT_ABS" "$RESTORED_MAJOR")"+# A data directory can only be started by a server of its own major version - the classic+# "restored 17, started 18" failure. On the host that server is pg_ctl; with --combine-image it is+# the image that both folds the chain and, per the instructions below, boots it. Asked before+# pg_combinebackup runs so a mismatch surfaces here instead of as its unrelated-looking failure.+if [ -z "$COMBINE_IMAGE" ]; then+ SERVER_MAJOR="$(pg_major_from_banner "$(pg_ctl --version 2>/dev/null || true)")"+ SERVER_SOURCE="the PostgreSQL installation"+ SERVER_REMEDY="install PostgreSQL $RESTORED_MAJOR (or pass --pg-bin <dir> pointing at it)"+else+ SERVER_MAJOR="$(image_server_major)"+ SERVER_SOURCE="image '$COMBINE_IMAGE'"+ SERVER_REMEDY="pass --combine-image postgres:$RESTORED_MAJOR"+fi++if [ -n "$RESTORED_MAJOR" ] && [ -n "$SERVER_MAJOR" ] && [ "$RESTORED_MAJOR" != "$SERVER_MAJOR" ]; then+ echo "error: restored cluster is PostgreSQL $RESTORED_MAJOR but $SERVER_SOURCE provides PostgreSQL $SERVER_MAJOR" >&2+ echo " A cluster can only be started by its own major version - $SERVER_REMEDY and re-run." >&2+ exit 1+fi+# Refuse to restore onto an already-initialized cluster at the target path - e.g. a volume the# postgres image already booted once (PG 18 leaves it at <out>/<major>/docker). That is the exact# empty-DB trap: we would write a second cluster the server never serves.@@ -314,18 +494,6 @@ elsefichmod 700 "$DATA_DIR"-# A data directory can only be started by a server of its own major version - the classic-# "restored 17, started 18" failure.-if [ -z "$COMBINE_IMAGE" ]; then- # On the host, pg_ctl is the server that will run it, so we can check it now.- TOOL_MAJOR="$(pg_major_from_banner "$(pg_ctl --version 2>/dev/null || true)")"- if [ -n "$RESTORED_MAJOR" ] && [ -n "$TOOL_MAJOR" ] && [ "$RESTORED_MAJOR" != "$TOOL_MAJOR" ]; then- echo "error: restored cluster is PostgreSQL $RESTORED_MAJOR but the PostgreSQL binaries are $TOOL_MAJOR" >&2- echo " A cluster can only be started by its own major version - install PostgreSQL $RESTORED_MAJOR (or pass --pg-bin <dir> pointing at it) and re-run." >&2- exit 1- fi-fi-# 5. Wire up WAL replay / PITR when the bundle ships WAL. A per-backup restore# ships none - its combined directory is already consistent, so we stop here.WAL_DIR="$BUNDLE_DIR/wal"@@ -404,9 +572,11 @@ CONTAINER_PGDATA="$(container_pgdata "$RESTORED_MAJOR")"echoecho "Restore prepared at: $DATA_DIR"echo "Next steps:"-echo " 1. Ensure the directory is owned by the postgres OS user:"-echo " chown -R postgres:postgres '$DATA_DIR'"if [ -n "$COMBINE_IMAGE" ]; then+ # The image's postgres runs as uid 999; the host's own postgres user is a different account, so+ # naming it here would hand the cluster to the wrong owner.+ echo " 1. Give the cluster to the uid the postgres image runs as:"+ echo " sudo chown -R 999:999 '$DATA_DIR'"echo " 2. Start PostgreSQL ${RESTORED_MAJOR:-<major>} with a matching postgres:${RESTORED_MAJOR:-<major>} image"echo " (the image needs no extra tools - the host already decompressed the backup and WAL):"if [ "${RESTORED_MAJOR:-0}" -ge 18 ] 2>/dev/null; then@@ -421,7 +591,18 @@ elsePG_CTL="pg_ctl"[ -n "$PG_BIN" ] && PG_CTL="$PG_BIN/pg_ctl"+ echo " 1. Ensure the directory is owned by the postgres OS user:"+ echo " chown -R postgres:postgres '$DATA_DIR'"echo " 2. Start PostgreSQL against it (as the postgres user), e.g.:"echo " $PG_CTL -D '$DATA_DIR' start"echo " or point your server's data_directory at it."fi++MISSING_CLUSTER_CONFIG="$(list_missing_cluster_config)"++if [ -n "$MISSING_CLUSTER_CONFIG" ]; then+ echo+ echo " It will not start yet - see ACTION REQUIRED below."++ print_missing_config_instructions "$MISSING_CLUSTER_CONFIG"+fidiff --git a/frontend/src/features/backups/physical/lib/restoreCommands.test.ts b/frontend/src/features/backups/physical/lib/restoreCommands.test.tsindex eb054dc4..63d45a46 100644--- a/frontend/src/features/backups/physical/lib/restoreCommands.test.ts+++ b/frontend/src/features/backups/physical/lib/restoreCommands.test.ts@@ -101,6 +101,7 @@ describe('buildDockerScriptCommand', () => {});describe('buildManualSteps', () => {+ const configCheckTitle = "Check the cluster's configuration files";const base = {bundleUrl,outputDir: './databasus-restore',@@ -182,6 +183,56 @@ describe('buildManualSteps', () => {expect(latestRecovery?.code).not.toContain('recovery_target_time');});+ it('reports missing configuration files rather than writing them, for both environments', () => {+ const hostSteps = buildManualSteps({ ...base, environment: 'host', hasWal: false });+ const dockerSteps = buildManualSteps({ ...base, environment: 'docker', hasWal: false });+ const hostCheckStep = hostSteps.find((step) => step.title === configCheckTitle);+ const dockerCheckStep = dockerSteps.find((step) => step.title === configCheckTitle);++ for (const checkStep of [hostCheckStep, dockerCheckStep]) {+ expect(checkStep?.code).toContain('echo "MISSING: $f"');+ expect(checkStep?.code).toContain('SHOW config_file');+ // Reporting only - the restore never invents an access-control policy for the user.+ expect(checkStep?.code).not.toContain('scram-sha-256');+ expect(checkStep?.code).not.toContain('trust');+ }+ });++ it('names the version-specific cluster and Debian configuration directories', () => {+ const pg17Steps = buildManualSteps({+ ...base,+ pgVersion: '17',+ environment: 'host',+ hasWal: false,+ });+ const pg18Steps = buildManualSteps({+ ...base,+ pgVersion: '18',+ environment: 'host',+ hasWal: false,+ });+ const pg17CheckStep = pg17Steps.find((step) => step.title === configCheckTitle);+ const pg18CheckStep = pg18Steps.find((step) => step.title === configCheckTitle);++ expect(pg17CheckStep?.code).toContain('"./databasus-restore/data/$f"');+ expect(pg17CheckStep?.code).toContain('/etc/postgresql/17/main');+ expect(pg18CheckStep?.code).toContain('"./databasus-restore/18/docker/$f"');+ expect(pg18CheckStep?.code).toContain('/etc/postgresql/18/main');+ });++ it('checks the configuration after reconstruction and before recovery is wired up', () => {+ const titles = buildManualSteps({ ...base, environment: 'host', hasWal: true }).map(+ (step) => step.title,+ );++ expect(titles.indexOf(configCheckTitle)).toBeGreaterThan(+ titles.indexOf('Reconstruct the data directory'),+ );+ expect(titles.indexOf(configCheckTitle)).toBeLessThan(+ titles.indexOf('Decompress WAL and wire up recovery'),+ );+ });+it('decompresses on the host then runs pg_combinebackup through docker in the docker environment', () => {const steps = buildManualSteps({ ...base, environment: 'docker', hasWal: false });const combine = steps.find((step) => step.title === 'Reconstruct the data directory');diff --git a/frontend/src/features/backups/physical/lib/restoreCommands.ts b/frontend/src/features/backups/physical/lib/restoreCommands.tsindex 0cbd225a..94693775 100644--- a/frontend/src/features/backups/physical/lib/restoreCommands.ts+++ b/frontend/src/features/backups/physical/lib/restoreCommands.ts@@ -24,6 +24,11 @@ export const containerVolumeDir = (pgVersion: string): string =>export const clusterDataDir = (outputDir: string, pgVersion: string): string =>Number(pgVersion) >= 18 ? `${outputDir}/${pgVersion}/docker` : `${outputDir}/data`;+// Outside the data directory, which is why a base backup never contains these files. The cluster is+// conventionally named "main" but pg_createcluster allows any name, so this is the convention rather+// than a fact about the user's source.+export const debianConfigDir = (pgVersion: string): string => `/etc/postgresql/${pgVersion}/main`;+export interface ScriptCommandParams {scriptUrl: string;bundleUrl: string;@@ -88,6 +93,34 @@ const decompressBackupsBlock = (): string =>`done`,].join('\n');+interface ClusterConfigCheckParams {+ dataDir: string;+ pgVersion: string;+}++// Mirrors the check the served recovery_script.sh runs - the two must stay in sync. Reports rather+// than repairs: pg_hba.conf is an access-control policy for the restored data, so which rules apply+// is the user's call, not ours.+const checkClusterConfigBlock = ({ dataDir, pgVersion }: ClusterConfigCheckParams): string =>+ [+ `for f in postgresql.conf pg_hba.conf pg_ident.conf; do`,+ ` [ -f "${dataDir}/$f" ] || echo "MISSING: $f"`,+ `done`,+ `# Nothing printed? Skip the rest of this step.`,+ `# A base backup copies only the data directory. Debian/Ubuntu keep the configuration in`,+ `# ${debianConfigDir(pgVersion)}, outside it, so it is not in the backup and never can be.`,+ `# Ask the source cluster where its own files are (needs superuser or pg_read_all_settings):`,+ `# psql -h <source-host> -U postgres -Atc "SHOW config_file"`,+ `# psql -h <source-host> -U postgres -Atc "SHOW hba_file"`,+ `# psql -h <source-host> -U postgres -Atc "SHOW ident_file"`,+ `# Copy all three into "${dataDir}/" (sudo if they are root-only), then in postgresql.conf:`,+ `# comment out data_directory, hba_file, ident_file, external_pid_file`,+ `# comment out ssl, ssl_cert_file, ssl_key_file, ssl_ca_file, or copy the certs too`,+ `# ("ssl = on" on its own fails as well - it then looks for server.crt in the data directory)`,+ `# mkdir "${dataDir}/conf.d" for the include_dir Debian ships`,+ `# set listen_addresses = '*' when the cluster runs in a container`,+ ].join('\n');+// `bundle/recon/full` plus every `bundle/recon/incr-N` in numeric order - the inputs// pg_combinebackup needs, oldest to newest. Output goes to the version-specific cluster dir.const combineCommand = (pgBin: string, dataDir: string): string =>@@ -182,6 +215,10 @@ export const buildManualSteps = ({title: 'Reconstruct the data directory',code: combine,},+ {+ title: "Check the cluster's configuration files",+ code: checkClusterConfigBlock({ dataDir, pgVersion }),+ },];if (hasWal) {diff --git a/frontend/src/features/backups/physical/ui/PhysicalRestoreComponent.tsx b/frontend/src/features/backups/physical/ui/PhysicalRestoreComponent.tsxindex 0b4615b5..60485a72 100644--- a/frontend/src/features/backups/physical/ui/PhysicalRestoreComponent.tsx+++ b/frontend/src/features/backups/physical/ui/PhysicalRestoreComponent.tsx@@ -18,6 +18,7 @@ import {clusterDataDir,containerDataDir,containerVolumeDir,+ debianConfigDir,} from '../lib/restoreCommands';interface Props {@@ -42,6 +43,24 @@ const describeRestoreError = (message: string): string => {return message;};+// Rendered for host and Docker alike: whether the source keeps its configuration outside the data+// directory cannot be known from here, and the resulting startup failure looks identical to a+// wrong-directory mount, so the difference has to be spelled out.+const missingConfigFilesNote = (pgVersion: string): JSX.Element => (+ <li>+ A base backup copies only the data directory. If your source PostgreSQL keeps its configuration+ outside it - the Debian/Ubuntu layout, <code>{debianConfigDir(pgVersion)}</code> - then{' '}+ <code>postgresql.conf</code>, <code>pg_hba.conf</code> and <code>pg_ident.conf</code> are not in+ the backup and the server will not start. Ask the source for their real paths with{' '}+ <code>psql -Atc "SHOW config_file"</code>, copy all three in, then comment out{' '}+ <code>data_directory</code>, <code>hba_file</code>, <code>ident_file</code>,{' '}+ <code>external_pid_file</code> and the <code>ssl</code> / <code>ssl_*</code> lines, create{' '}+ <code>conf.d</code>, and set <code>listen_addresses = '*'</code> for a container. If+ those files are present and you still get the same error, it is the mounted directory that is+ wrong, not the configuration.+ </li>+);+interface CopyableCommandProps {id: string;title: string;@@ -280,6 +299,7 @@ export const PhysicalRestoreComponent = ({ database, backup, onClose }: Props):PostgreSQL replays WAL and promotes - watch the log for recovery completion.</li>)}+ {missingConfigFilesNote(pgVersion)}</ul>) : (<ul className="ml-4 list-disc">@@ -308,7 +328,10 @@ export const PhysicalRestoreComponent = ({ database, backup, onClose }: Props):</code></li>)}- <li>Ownership must match the postgres uid inside the image.</li>+ <li>+ Ownership must match the postgres uid inside the image (999 in the official one).+ </li>+ {missingConfigFilesNote(pgVersion)}</ul>)}