Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(scripts): snapshot every restore target before writing any

Yudhi Armyndharis4e9793752fa7
diff --git a/docs/11-operational-runbooks.md b/docs/11-operational-runbooks.md
index 6d737e66f..31f7dbd8d 100644
--- a/docs/11-operational-runbooks.md
+++ b/docs/11-operational-runbooks.md
@@ -678,10 +678,10 @@ curl -s -X POST -H "X-API-Key: <an-existing-key>" http://localhost:2785/api/auth
> file keeps the data in the **named volume** `openwa-data` and the Helm chart in a PVC, so a host run
> there fills a directory the container never reads and still reports success. Run the script from the
> image against the volume instead, in place of step 2. Both mount the container root read-only, so
-> `OPENWA_RESTORE_SNAPSHOT_DIR` (0.23.7 or later) puts the pre-restore snapshot of the data dir on a
-> writable, persistent path, and `TMPDIR` keeps the extracted archive there too; allow free space for
-> about twice the data plus the archive. `--force` is included because the volume of an existing
-> install still holds its databases:
+> `OPENWA_RESTORE_SNAPSHOT_DIR` (0.23.7 or later) puts the pre-restore snapshots, of the data dir and of
+> any state directory mounted outside it, on a writable, persistent path, and `TMPDIR` keeps the
+> extracted archive there too; allow free space for about twice the data plus the archive. `--force` is
+> included because the volume of an existing install still holds its databases:
>
> ```bash
> # Compose: the entrypoint override runs the script as root, which can read the archive and write
@@ -724,7 +724,7 @@ curl -s -X POST -H "X-API-Key: <an-existing-key>" http://localhost:2785/api/auth
> # HOME is moved off it here (the compose files already set HOME=/tmp).
> kubectl exec openwa-restore -- env HOME=/tmp OPENWA_RESTORE_SNAPSHOT_DIR=/restore TMPDIR=/restore \
> ./scripts/restore.sh /restore/backup.tar.gz --force
-> # The emptyDir goes away with the pod: copy off the snapshot the script named first.
+> # The emptyDir goes away with the pod: copy off every snapshot the script named first.
> kubectl cp openwa-restore:/restore/data.pre-restore-<ts> ./backups/data.pre-restore-<ts>
> kubectl delete pod openwa-restore
> kubectl scale statefulset/openwa --replicas=1
diff --git a/scripts/restore.sh b/scripts/restore.sh
index 19baf04b3..0c823e81b 100755
--- a/scripts/restore.sh
+++ b/scripts/restore.sh
@@ -25,11 +25,12 @@
# SESSION_DATA_PATH, BAILEYS_AUTH_DIR, STORAGE_LOCAL_PATH, PLUGINS_DIR
# override the corresponding state directories
# OPENWA_RESTORE_SNAPSHOT_DIR
-# where the data-dir safety snapshot goes (default: next to the data dir);
-# needed when that parent is read-only, as in the shipped container
+# where the safety snapshots go (default: next to the data dir, and next to
+# each target outside it); needed when a parent is read-only, as in the
+# shipped container
#
-# Stop the OpenWA app before restoring. A snapshot of the current data dir is taken
-# first so a bad restore can be undone.
+# Stop the OpenWA app before restoring. A snapshot of the current data dir, and of every target
+# outside it, is taken before anything is written so a bad restore can be undone.
#
set -euo pipefail
# Restored databases, credentials, and snapshots must not inherit a permissive operator umask.
@@ -162,40 +163,53 @@ replace_tree() {
exit 1
;;
esac
- # The initial data-dir snapshot already covers normal nested targets. Preserve any custom target
- # outside data/ separately before replacing it, so a custom PLUGINS_DIR/auth path remains recoverable.
- case "$resolved_target" in
- "$RESOLVED_DATA_DIR"/*) ;;
- *)
- if [ -e "$target_dir" ]; then
- external_snapshot="${target_dir%/}.pre-restore-$RESTORE_TIMESTAMP"
- log "Snapshotting current $label -> $external_snapshot"
- cp -pR "$target_dir" "$external_snapshot"
- fi
- ;;
- esac
+ if [ "$PHASE" = snapshot ]; then
+ snapshot_external "$target_dir"
+ return
+ fi
log "Restoring $label"
rm -rf -- "$target_dir"
mkdir -p "$(dirname "$target_dir")"
cp -pR "$source_dir" "$target_dir"
}
-# The data-dir safety snapshot below cannot cover a database target that lives OUTSIDE it (custom
-# MAIN_DATABASE_NAME / DATABASE_NAME). Preserve such a file separately before overwriting it, so a
-# restore pointed at the wrong archive remains recoverable.
-snapshot_external_db() {
+# The data-dir safety snapshot below cannot cover a target that lives OUTSIDE it (a custom
+# MAIN_DATABASE_NAME / DATABASE_NAME, SESSION_DATA_PATH, BAILEYS_AUTH_DIR, STORAGE_LOCAL_PATH or
+# PLUGINS_DIR). Preserve such a target separately, so a restore pointed at the wrong archive remains
+# recoverable. OPENWA_RESTORE_SNAPSHOT_DIR takes these snapshots too when it is set: a target on its
+# own mount under a read-only root has no writable place next to it.
+snapshot_external() {
target="$1"
- resolved_target="$(resolve_path "$target")"
- case "$resolved_target" in
- "$RESOLVED_DATA_DIR"/*) ;;
- *)
- if [ -e "$target" ]; then
- external_snapshot="${target}.pre-restore-$RESTORE_TIMESTAMP"
- log "Snapshotting current $target -> $external_snapshot"
- cp -p "$target" "$external_snapshot"
- fi
- ;;
+ case "$(resolve_path "$target")" in
+ "$RESOLVED_DATA_DIR"/*) return 0 ;;
esac
+ [ -e "$target" ] || return 0
+ snapshot_dir="${OPENWA_RESTORE_SNAPSHOT_DIR:-$(dirname "$target")}"
+ external_snapshot="${snapshot_dir%/}/$(basename "$target").pre-restore-$RESTORE_TIMESTAMP"
+ # Targets from different parents can share a name in one snapshot dir (a PLUGINS_DIR and a
+ # PLUGIN_STATE_DIR both end in plugins); never copy one into the other's snapshot.
+ n=1
+ while [ -e "$external_snapshot" ]; do
+ n=$((n + 1))
+ external_snapshot="${snapshot_dir%/}/$(basename "$target").pre-restore-$RESTORE_TIMESTAMP-$n"
+ done
+ log "Snapshotting current $target -> $external_snapshot"
+ mkdir -p "$snapshot_dir"
+ cp -pR "$target" "$external_snapshot"
+}
+
+# restore_db <staged file> <target> <label>
+restore_db() {
+ if [ "$PHASE" = snapshot ]; then
+ snapshot_external "$2"
+ return
+ fi
+ log "Restoring $3 -> $2"
+ mkdir -p "$(dirname "$2")"
+ cp "$1" "$2"
+ # Owner-only, matching what the app re-tightens on every boot (sqlite-file-permissions.ts);
+ # cp preserves the staged mode, and a foreign-umask extraction may leave it broader.
+ chmod 0600 "$2" 2>/dev/null || true
}
# A restore target that already holds a working install's tables is LIVE: overwriting it destroys
@@ -289,58 +303,56 @@ fi
mkdir -p "$DATA_DIR"
-if [ -f "$STAGE/main.sqlite" ]; then
- log "Restoring auth/audit DB -> $MAIN_DB"
- snapshot_external_db "$MAIN_DB"
- mkdir -p "$(dirname "$MAIN_DB")"
- cp "$STAGE/main.sqlite" "$MAIN_DB"
- # Owner-only, matching what the app re-tightens on every boot (sqlite-file-permissions.ts);
- # cp preserves the staged mode, and a foreign-umask extraction may leave it broader.
- chmod 0600 "$MAIN_DB" 2>/dev/null || true
-else
+if [ ! -f "$STAGE/main.sqlite" ]; then
log "WARN: main.sqlite not in archive — API keys / audit log will NOT be restored"
fi
-if [ -f "$STAGE/openwa.sqlite" ]; then
- log "Restoring data store -> $DATA_DB"
- snapshot_external_db "$DATA_DB"
- mkdir -p "$(dirname "$DATA_DB")"
- cp "$STAGE/openwa.sqlite" "$DATA_DB"
- chmod 0600 "$DATA_DB" 2>/dev/null || true
+MERGED_PLUGINS_DIR=""
+if [ -d "$STAGE/plugin-packages" ] && [ -d "$STAGE/plugin-state" ] &&
+ [ "$(resolve_path "$PLUGIN_PACKAGES_DIR")" = "$(resolve_path "$PLUGIN_STATE_DIR")" ]; then
+ # Docker deployments deliberately colocate package and state files. Build the complete target in
+ # staging and replace it once, so neither half can erase the other during restore.
+ MERGED_PLUGINS_DIR="$STAGE/plugin-merged"
+ mkdir -p "$MERGED_PLUGINS_DIR"
+ cp -pR "$STAGE/plugin-packages/." "$MERGED_PLUGINS_DIR"
+ cp -pR "$STAGE/plugin-state/." "$MERGED_PLUGINS_DIR"
fi
-if [ -d "$STAGE/sessions" ]; then
- replace_tree "$STAGE/sessions" "$SESSIONS_DIR" "whatsapp-web.js sessions"
-fi
-
-if [ -d "$STAGE/baileys" ]; then
- replace_tree "$STAGE/baileys" "$BAILEYS_DIR" "Baileys authentication state"
-fi
-
-if [ -d "$STAGE/media" ]; then
- replace_tree "$STAGE/media" "$MEDIA_DIR" "local media"
-fi
-
-if [ -d "$STAGE/plugin-packages" ] && [ -d "$STAGE/plugin-state" ]; then
- RESOLVED_PLUGIN_PACKAGES_DIR="$(resolve_path "$PLUGIN_PACKAGES_DIR")"
- RESOLVED_PLUGIN_STATE_DIR="$(resolve_path "$PLUGIN_STATE_DIR")"
- if [ "$RESOLVED_PLUGIN_PACKAGES_DIR" = "$RESOLVED_PLUGIN_STATE_DIR" ]; then
- # Docker deployments deliberately colocate package and state files. Build the complete target in
- # staging and replace it once, so neither half can erase the other during restore.
- MERGED_PLUGINS_DIR="$STAGE/plugin-merged"
- mkdir -p "$MERGED_PLUGINS_DIR"
- cp -pR "$STAGE/plugin-packages/." "$MERGED_PLUGINS_DIR"
- cp -pR "$STAGE/plugin-state/." "$MERGED_PLUGINS_DIR"
+# Runs twice: PHASE=snapshot checks every target and snapshots the ones outside the data dir, then
+# PHASE=apply writes them. A target that cannot be snapshotted or is refused stops the restore
+# before the first database is written, instead of halfway through with a mixed install left behind.
+restore_targets() {
+ if [ -f "$STAGE/main.sqlite" ]; then
+ restore_db "$STAGE/main.sqlite" "$MAIN_DB" "auth/audit DB"
+ fi
+ if [ -f "$STAGE/openwa.sqlite" ]; then
+ restore_db "$STAGE/openwa.sqlite" "$DATA_DB" "data store"
+ fi
+ if [ -d "$STAGE/sessions" ]; then
+ replace_tree "$STAGE/sessions" "$SESSIONS_DIR" "whatsapp-web.js sessions"
+ fi
+ if [ -d "$STAGE/baileys" ]; then
+ replace_tree "$STAGE/baileys" "$BAILEYS_DIR" "Baileys authentication state"
+ fi
+ if [ -d "$STAGE/media" ]; then
+ replace_tree "$STAGE/media" "$MEDIA_DIR" "local media"
+ fi
+ if [ -n "$MERGED_PLUGINS_DIR" ]; then
replace_tree "$MERGED_PLUGINS_DIR" "$PLUGIN_PACKAGES_DIR" "installed plugins and plugin state"
else
- replace_tree "$STAGE/plugin-packages" "$PLUGIN_PACKAGES_DIR" "installed plugin packages"
- replace_tree "$STAGE/plugin-state" "$PLUGIN_STATE_DIR" "plugin registry and persisted state"
+ if [ -d "$STAGE/plugin-packages" ]; then
+ replace_tree "$STAGE/plugin-packages" "$PLUGIN_PACKAGES_DIR" "installed plugin packages"
+ fi
+ if [ -d "$STAGE/plugin-state" ]; then
+ replace_tree "$STAGE/plugin-state" "$PLUGIN_STATE_DIR" "plugin registry and persisted state"
+ fi
fi
-elif [ -d "$STAGE/plugin-packages" ]; then
- replace_tree "$STAGE/plugin-packages" "$PLUGIN_PACKAGES_DIR" "installed plugin packages"
-elif [ -d "$STAGE/plugin-state" ]; then
- replace_tree "$STAGE/plugin-state" "$PLUGIN_STATE_DIR" "plugin registry and persisted state"
-fi
+}
+
+PHASE=snapshot
+restore_targets
+PHASE=apply
+restore_targets
if [ -f "$STAGE/.env.generated" ]; then
log "Restoring dashboard-generated configuration"
diff --git a/scripts/smoke-test-backup-restore.sh b/scripts/smoke-test-backup-restore.sh
index 0242d9a12..3187e735c 100755
--- a/scripts/smoke-test-backup-restore.sh
+++ b/scripts/smoke-test-backup-restore.sh
@@ -17,6 +17,8 @@
# (k) an operator's sqlite3 rc file changes neither answer of the guard (skipped without sqlite3)
# (l) an unwritable BACKUP_DIR fails before anything is staged (skipped as root)
# (m) OPENWA_RESTORE_SNAPSHOT_DIR takes the data-dir snapshot off a read-only parent (skipped as root)
+# (n) a state dir outside the data dir is snapshotted before any database is written, and under
+# OPENWA_RESTORE_SNAPSHOT_DIR when that is set (skipped as root)
#
# Usage: ./scripts/smoke-test-backup-restore.sh
# Requires: bash, tar, node (restore.sh path resolution). sqlite3 is optional (see (c) and (k)).
@@ -531,5 +533,68 @@ else
echo "SKIP: (m) running as root, which ignores the permission bits this case relies on"
fi
+echo ""
+echo "==> (n) a state dir outside the data dir is snapshotted before any database is written"
+# SESSION_DATA_PATH on its own mount has a parent of its own. When that parent is read-only, the
+# snapshot of the directory cannot go next to it, and a restore that finds out only after writing
+# the databases leaves them from the archive and the sessions from the live install.
+if [ "$(id -u)" -ne 0 ]; then
+ N="$WORK/n"
+ mkdir -p "$N/src/data/sessions/session-s1" "$N/live" "$N/ro/sessions/session-s1" "$N/ext/sessions/session-s1"
+ make_fixture "$N/src/data/main.sqlite" "november-archive-main"
+ make_fixture "$N/src/data/openwa.sqlite" "november-archive-data"
+ printf 'november-archive\n' >"$N/src/data/sessions/session-s1/marker"
+ (
+ cd "$N/src"
+ BACKUP_DIR="$N/out" "$BACKUP" >/dev/null
+ )
+ ARCHIVE_N="$(ls "$N"/out/openwa-backup-*.tar.gz)"
+ make_fixture "$N/live/main.sqlite" "november-live-main"
+ make_fixture "$N/live/openwa.sqlite" "november-live-data"
+ printf 'november-live\n' >"$N/ro/sessions/session-s1/marker"
+ printf 'november-live\n' >"$N/ext/sessions/session-s1/marker"
+
+ # restore_n <sessions dir> [snapshot dir]: a forced restore of ARCHIVE_N over $N/live. Output lands
+ # in OUT, the exit code in RC.
+ restore_n() {
+ set +e
+ OUT="$(cd "$N" && MAIN_DATABASE_NAME="$N/live/main.sqlite" DATABASE_NAME="$N/live/openwa.sqlite" \
+ OPENWA_DATA_DIR="$N/live" SESSION_DATA_PATH="$1" OPENWA_RESTORE_SNAPSHOT_DIR="${2:-}" \
+ "$RESTORE" "$ARCHIVE_N" --force 2>&1)"
+ RC=$?
+ set -e
+ }
+
+ chmod a-w "$N/ro"
+ restore_n "$N/ro/sessions"
+ chmod u+w "$N/ro"
+ if [ "$RC" -eq 0 ]; then
+ fail "(n) restore exited 0 although the sessions snapshot could not be written"
+ fi
+ if [ "$(db_fingerprint "$N/live/main.sqlite")" != "november-live-main" ]; then
+ fail "(n) the main DB was overwritten before the sessions snapshot failed"
+ fi
+ if [ "$(cat "$N/ro/sessions/session-s1/marker")" != "november-live" ]; then
+ fail "(n) the failed restore changed the live sessions"
+ fi
+
+ restore_n "$N/ext/sessions" "$N/snapshots"
+ if [ "$RC" -ne 0 ]; then
+ fail "(n) restore with an external SESSION_DATA_PATH failed: $OUT"
+ fi
+ if [ "$(cat "$N"/snapshots/sessions.pre-restore-*/session-s1/marker 2>/dev/null || true)" != "november-live" ]; then
+ fail "(n) the sessions snapshot is not under OPENWA_RESTORE_SNAPSHOT_DIR"
+ fi
+ if [ -n "$(ls -d "$N"/ext/sessions.pre-restore-* 2>/dev/null || true)" ]; then
+ fail "(n) the sessions snapshot was written next to the target despite OPENWA_RESTORE_SNAPSHOT_DIR"
+ fi
+ if [ "$(cat "$N/ext/sessions/session-s1/marker")" != "november-archive" ]; then
+ fail "(n) the archived sessions were not restored"
+ fi
+ pass "(n) external state snapshotted before the databases are written, under OPENWA_RESTORE_SNAPSHOT_DIR"
+else
+ echo "SKIP: (n) running as root, which ignores the permission bits this case relies on"
+fi
+
echo ""
echo "All smoke tests passed!"