Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (restore): Pass each pg_combinebackup input directory as one argument, so a restore into a path with a space no longer fails, and delete the stale root copy of the recovery script

Rostislav Duginec7ee41df13c
diff --git a/backend/internal/features/backups/backups/controllers/physical/recovery_script.sh b/backend/internal/features/backups/backups/controllers/physical/recovery_script.sh
index 28046b15..a416bc0f 100644
--- a/backend/internal/features/backups/backups/controllers/physical/recovery_script.sh
+++ b/backend/internal/features/backups/backups/controllers/physical/recovery_script.sh
@@ -416,7 +416,9 @@ echo "Reconstructing data directory with pg_combinebackup..."
# shellcheck disable=SC2012 # the backend names the bundle directories without spaces
INCR_DIRS="$(cd "$BUNDLE_DIR" && ls -d incr-* 2>/dev/null | sort -V || true)"
RECON_DIR="$BUNDLE_DIR/recon"
-INPUTS=""
+# POSIX sh has no arrays, so the positional parameters collect pg_combinebackup's input directories:
+# a directory whose path holds a space stays one argument. The options were parsed above.
+set --
for backup in full $INCR_DIRS; do
src="$BUNDLE_DIR/$backup"
dst="$RECON_DIR/$backup"
@@ -444,7 +446,7 @@ for backup in full $INCR_DIRS; do
esac
cp "$src/backup_manifest" "$dst/backup_manifest"
- INPUTS="$INPUTS $dst"
+ set -- "$@" "$dst"
done
# The backup's major version is its PG_VERSION (host-side file, so this works in --combine-image
@@ -487,14 +489,12 @@ rm -rf "$DATA_DIR"
if [ -n "$COMBINE_IMAGE" ]; then
# Run pg_combinebackup in the container as the host user (so the output stays
# host-owned and the later chmod / WAL-wiring on the host can touch it) and bind
- # the output dir at its own absolute path so $INPUTS / $DATA_DIR resolve the same
- # inside and out.
- # shellcheck disable=SC2086 # INPUTS holds one directory per word, so an output path with a space breaks the restore
+ # the output dir at its own absolute path so the input directories and $DATA_DIR resolve the
+ # same inside and out.
docker run --rm --user "$(id -u):$(id -g)" -v "$OUT_ABS:$OUT_ABS" -w "$OUT_ABS" \
- "$COMBINE_IMAGE" pg_combinebackup $INPUTS -o "$DATA_DIR"
+ "$COMBINE_IMAGE" pg_combinebackup "$@" -o "$DATA_DIR"
else
- # shellcheck disable=SC2086 # INPUTS holds one directory per word, so an output path with a space breaks the restore
- pg_combinebackup $INPUTS -o "$DATA_DIR"
+ pg_combinebackup "$@" -o "$DATA_DIR"
fi
chmod 700 "$DATA_DIR"
diff --git a/backend/internal/features/tests/physical/postgresql/shared/helpers.go b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
index 8ae09d79..50b8ec3c 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/helpers.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
@@ -143,7 +143,9 @@ func runRecoveryScriptOnHost(
// reconstructOnHost runs the recovery script on the host (--combine-image mode) into a fresh output
// dir and returns it. Separate from the boot step so callers can probe the re-restore guard while the
-// output is still test-owned (the entrypoint chowns PGDATA to the postgres uid once it boots).
+// output is still test-owned (the entrypoint chowns PGDATA to the postgres uid once it boots). The
+// output dir's name holds a space, as a user's might, so every path the script passes to
+// pg_combinebackup must stay one argument.
func reconstructOnHost(
t *testing.T,
router *gin.Engine,
@@ -153,7 +155,7 @@ func reconstructOnHost(
t.Helper()
script := fetchRecoveryScript(t, router)
- outDir := t.TempDir()
+ outDir := filepath.Join(t.TempDir(), "restored cluster")
out, err := runRecoveryScriptOnHost(t, script, bundle, outDir, image, targetTime)
require.NoError(t, err, "recovery script must succeed into an empty dir:\n%s", out)
diff --git a/databasus-recovery.sh b/databasus-recovery.sh
deleted file mode 100644
index 3817ee05..00000000
--- a/databasus-recovery.sh
+++ /dev/null
@@ -1,239 +0,0 @@
-#!/bin/sh
-# Databasus physical-restore helper.
-#
-# Takes a restore bundle produced by the Databasus restore stream, decompresses
-# what needs decompressing, runs pg_combinebackup over the full + incremental
-# chain, and wires up WAL replay / point-in-time recovery.
-#
-# Usage:
-# sh databasus-recovery.sh [--pg-bin <dir>] [--target-time <utc-ts>] <bundle-url-or-path> [output-dir]
-#
-# --pg-bin <dir> directory holding the PostgreSQL tools (pg_combinebackup,
-# pg_ctl) when they are not on PATH.
-# --target-time <ts> point-in-time recovery target, a PostgreSQL-parseable UTC
-# timestamp (e.g. '2026-06-12 14:30:00+00:00'). Replays WAL
-# up to this point and promotes; omit to replay all shipped
-# WAL (latest). Ignored when the bundle ships no WAL.
-# <bundle-url-or-path> an http(s):// restore-stream URL (with its token) or a
-# path to an already-downloaded .tar bundle.
-# [output-dir] where to build the restore (default: databasus-restore).
-
-set -eu
-
-usage() {
- cat >&2 <<'USAGE'
-usage: sh databasus-recovery.sh [--pg-bin <dir>] [--target-time <utc-ts>] <bundle-url-or-path> [output-dir]
-
- --pg-bin <dir> directory holding the PostgreSQL tools (pg_combinebackup,
- pg_ctl) when they are not on PATH
- --target-time <ts> point-in-time recovery target, a PostgreSQL-parseable UTC
- timestamp (e.g. '2026-06-12 14:30:00+00:00'); omit to
- replay all shipped WAL. Ignored when the bundle ships no WAL
- <bundle-url-or-path> an http(s):// restore-stream URL (with its token) or a
- path to an already-downloaded .tar bundle
- [output-dir] where to build the restore (default: databasus-restore)
-USAGE
-}
-
-PG_BIN=""
-TARGET_TIME=""
-while [ $# -gt 0 ]; do
- case "$1" in
- --pg-bin)
- [ $# -ge 2 ] || {
- echo "error: --pg-bin needs a directory" >&2
- exit 2
- }
- PG_BIN="$2"
- shift 2
- ;;
- --pg-bin=*)
- PG_BIN="${1#*=}"
- shift
- ;;
- --target-time)
- [ $# -ge 2 ] || {
- echo "error: --target-time needs a timestamp" >&2
- exit 2
- }
- TARGET_TIME="$2"
- shift 2
- ;;
- --target-time=*)
- TARGET_TIME="${1#*=}"
- shift
- ;;
- -h | --help)
- usage
- exit 0
- ;;
- --)
- shift
- break
- ;;
- -*)
- echo "error: unknown option '$1'" >&2
- usage
- exit 2
- ;;
- *)
- break
- ;;
- esac
-done
-
-SOURCE="${1:-}"
-OUT_DIR="${2:-databasus-restore}"
-
-if [ -z "$SOURCE" ]; then
- usage
- exit 2
-fi
-
-require_tool() {
- if ! command -v "$1" >/dev/null 2>&1; then
- echo "error: required tool '$1' is not installed or not on PATH" >&2
- exit 1
- fi
-}
-
-# Resolve the PostgreSQL tools off --pg-bin when they are not on PATH; every later
-# pg_combinebackup / pg_ctl lookup then finds them.
-if [ -n "$PG_BIN" ]; then
- if [ ! -x "$PG_BIN/pg_combinebackup" ]; then
- echo "error: --pg-bin '$PG_BIN' has no executable pg_combinebackup" >&2
- exit 1
- fi
-
- PATH="$PG_BIN:$PATH"
- export PATH
-fi
-
-require_tool tar
-require_tool pg_combinebackup
-
-mkdir -p "$OUT_DIR"
-OUT_ABS="$(cd "$OUT_DIR" && pwd)"
-BUNDLE_DIR="$OUT_ABS/bundle"
-DATA_DIR="$OUT_ABS/data"
-mkdir -p "$BUNDLE_DIR"
-
-# 1. Obtain the bundle.
-case "$SOURCE" in
- http://* | https://*)
- TAR_PATH="$OUT_ABS/restore.tar"
- echo "Downloading restore bundle..."
- if command -v curl >/dev/null 2>&1; then
- curl -fSL "$SOURCE" -o "$TAR_PATH"
- elif command -v wget >/dev/null 2>&1; then
- wget -O "$TAR_PATH" "$SOURCE"
- else
- echo "error: need curl or wget to download $SOURCE" >&2
- exit 1
- fi
- ;;
- *)
- TAR_PATH="$SOURCE"
- if [ ! -f "$TAR_PATH" ]; then
- echo "error: bundle file not found: $TAR_PATH" >&2
- exit 1
- fi
- ;;
-esac
-
-# 2. Extract.
-echo "Extracting bundle..."
-tar -xf "$TAR_PATH" -C "$BUNDLE_DIR"
-
-# 3. Verify the transfer before touching the bytes (compressed WAL included).
-if [ -f "$BUNDLE_DIR/MANIFEST.sha256" ] && command -v sha256sum >/dev/null 2>&1; then
- echo "Verifying checksums..."
- (cd "$BUNDLE_DIR" && sha256sum -c MANIFEST.sha256 >/dev/null) ||
- {
- echo "error: checksum verification failed" >&2
- exit 1
- }
-fi
-
-# 4. Decompress each backup blob into its own directory, then fold the full +
-# incremental chain (oldest -> newest) with pg_combinebackup. Each backup ships as
-# <dir>/base.tar<ext> (still compressed) plus its <dir>/backup_manifest sidecar;
-# pg_combinebackup reads the manifest from the extracted directory.
-echo "Reconstructing data directory with pg_combinebackup..."
-# shellcheck disable=SC2012 # the backend names the bundle directories without spaces
-INCR_DIRS="$(cd "$BUNDLE_DIR" && ls -d incr-* 2>/dev/null | sort -V || true)"
-RECON_DIR="$BUNDLE_DIR/recon"
-INPUTS=""
-for backup in full $INCR_DIRS; do
- src="$BUNDLE_DIR/$backup"
- dst="$RECON_DIR/$backup"
- mkdir -p "$dst"
-
- # shellcheck disable=SC2012 # the backend names the payload files without spaces
- blob="$(ls "$src"/base.tar* 2>/dev/null | head -n1 || true)"
- if [ -z "$blob" ]; then
- echo "error: backup '$backup' has no base.tar payload" >&2
- exit 1
- fi
-
- case "$blob" in
- *.zst)
- require_tool zstd
- zstd -dqc "$blob" | tar -xf - -C "$dst"
- ;;
- *.gz)
- gzip -dc "$blob" | tar -xf - -C "$dst"
- ;;
- *)
- tar -xf "$blob" -C "$dst"
- ;;
- esac
-
- cp "$src/backup_manifest" "$dst/backup_manifest"
- INPUTS="$INPUTS $dst"
-done
-
-rm -rf "$DATA_DIR"
-# shellcheck disable=SC2086 # INPUTS holds one directory per word, so an output path with a space breaks the restore
-pg_combinebackup $INPUTS -o "$DATA_DIR"
-chmod 700 "$DATA_DIR"
-
-# 5. Wire up WAL replay / PITR when the bundle ships WAL. A per-backup restore
-# ships none - its combined directory is already consistent, so we stop here.
-WAL_DIR="$BUNDLE_DIR/wal"
-if [ -d "$WAL_DIR" ] && [ -n "$(ls -A "$WAL_DIR" 2>/dev/null)" ]; then
- # WAL segments ship zstd-compressed and are inflated on demand at replay time.
- require_tool zstd
- ZSTD_BIN="$(command -v zstd)"
-
- # PostgreSQL runs restore_command through `sh -c` with %f/%p substituted, so
- # inline the lookup: decompress a .zst segment on the fly, fall back to a
- # plaintext file (history files), or exit non-zero to signal end-of-archive.
- # Inlining (rather than a helper script in PGDATA) avoids depending on the
- # data directory being on an exec-mounted filesystem. Only double quotes are
- # used inside so the value stays safely wrapped in single quotes.
- restore_command="d=\"$WAL_DIR\"; if [ -f \"\$d/%f.zst\" ]; then \"$ZSTD_BIN\" -dqc \"\$d/%f.zst\" > %p; elif [ -f \"\$d/%f\" ]; then cp \"\$d/%f\" %p; else exit 1; fi"
-
- {
- echo "archive_mode = off"
- echo "archive_command = ''"
- echo "restore_command = '$restore_command'"
- if [ -n "$TARGET_TIME" ]; then
- echo "recovery_target_time = '$TARGET_TIME'"
- echo "recovery_target_action = 'promote'"
- fi
- } >>"$DATA_DIR/postgresql.auto.conf"
- touch "$DATA_DIR/recovery.signal"
-fi
-
-PG_CTL="pg_ctl"
-[ -n "$PG_BIN" ] && PG_CTL="$PG_BIN/pg_ctl"
-
-echo
-echo "Restore prepared at: $DATA_DIR"
-echo "Next steps:"
-echo " 1. Ensure the directory is owned by the postgres OS user:"
-echo " chown -R postgres:postgres '$DATA_DIR'"
-echo " 2. Start PostgreSQL against it (as the postgres user), e.g.:"
-echo " $PG_CTL -D '$DATA_DIR' start"
-echo " or point your server's data_directory at it."