Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(scripts): snapshot every restore target before writing any
Yudhi Armyndharis4e9793752fa7
diff --git a/docs/11-operational-runbooks.md b/docs/11-operational-runbooks.mdindex 6d737e66f..31f7dbd8d 100644--- a/docs/11-operational-runbooks.md+++ b/docs/11-operational-runbooks.md@@ -678,10 +678,10 @@ curl -s -X POST -H "X-API-Key: <an-existing-key>" http://localhost:2785/api/auth> file keeps the data in the **named volume** `openwa-data` and the Helm chart in a PVC, so a host run> there fills a directory the container never reads and still reports success. Run the script from the> image against the volume instead, in place of step 2. Both mount the container root read-only, so-> `OPENWA_RESTORE_SNAPSHOT_DIR` (0.23.7 or later) puts the pre-restore snapshot of the data dir on a-> writable, persistent path, and `TMPDIR` keeps the extracted archive there too; allow free space for-> about twice the data plus the archive. `--force` is included because the volume of an existing-> install still holds its databases:+> `OPENWA_RESTORE_SNAPSHOT_DIR` (0.23.7 or later) puts the pre-restore snapshots, of the data dir and of+> any state directory mounted outside it, on a writable, persistent path, and `TMPDIR` keeps the+> extracted archive there too; allow free space for about twice the data plus the archive. `--force` is+> included because the volume of an existing install still holds its databases:>> ```bash> # Compose: the entrypoint override runs the script as root, which can read the archive and write@@ -724,7 +724,7 @@ curl -s -X POST -H "X-API-Key: <an-existing-key>" http://localhost:2785/api/auth> # HOME is moved off it here (the compose files already set HOME=/tmp).> kubectl exec openwa-restore -- env HOME=/tmp OPENWA_RESTORE_SNAPSHOT_DIR=/restore TMPDIR=/restore \> ./scripts/restore.sh /restore/backup.tar.gz --force-> # The emptyDir goes away with the pod: copy off the snapshot the script named first.+> # The emptyDir goes away with the pod: copy off every snapshot the script named first.> kubectl cp openwa-restore:/restore/data.pre-restore-<ts> ./backups/data.pre-restore-<ts>> kubectl delete pod openwa-restore> kubectl scale statefulset/openwa --replicas=1diff --git a/scripts/restore.sh b/scripts/restore.shindex 19baf04b3..0c823e81b 100755--- a/scripts/restore.sh+++ b/scripts/restore.sh@@ -25,11 +25,12 @@# SESSION_DATA_PATH, BAILEYS_AUTH_DIR, STORAGE_LOCAL_PATH, PLUGINS_DIR# override the corresponding state directories# OPENWA_RESTORE_SNAPSHOT_DIR-# where the data-dir safety snapshot goes (default: next to the data dir);-# needed when that parent is read-only, as in the shipped container+# where the safety snapshots go (default: next to the data dir, and next to+# each target outside it); needed when a parent is read-only, as in the+# shipped container#-# Stop the OpenWA app before restoring. A snapshot of the current data dir is taken-# first so a bad restore can be undone.+# Stop the OpenWA app before restoring. A snapshot of the current data dir, and of every target+# outside it, is taken before anything is written so a bad restore can be undone.#set -euo pipefail# Restored databases, credentials, and snapshots must not inherit a permissive operator umask.@@ -162,40 +163,53 @@ replace_tree() {exit 1;;esac- # The initial data-dir snapshot already covers normal nested targets. Preserve any custom target- # outside data/ separately before replacing it, so a custom PLUGINS_DIR/auth path remains recoverable.- case "$resolved_target" in- "$RESOLVED_DATA_DIR"/*) ;;- *)- if [ -e "$target_dir" ]; then- external_snapshot="${target_dir%/}.pre-restore-$RESTORE_TIMESTAMP"- log "Snapshotting current $label -> $external_snapshot"- cp -pR "$target_dir" "$external_snapshot"- fi- ;;- esac+ if [ "$PHASE" = snapshot ]; then+ snapshot_external "$target_dir"+ return+ filog "Restoring $label"rm -rf -- "$target_dir"mkdir -p "$(dirname "$target_dir")"cp -pR "$source_dir" "$target_dir"}-# The data-dir safety snapshot below cannot cover a database target that lives OUTSIDE it (custom-# MAIN_DATABASE_NAME / DATABASE_NAME). Preserve such a file separately before overwriting it, so a-# restore pointed at the wrong archive remains recoverable.-snapshot_external_db() {+# The data-dir safety snapshot below cannot cover a target that lives OUTSIDE it (a custom+# MAIN_DATABASE_NAME / DATABASE_NAME, SESSION_DATA_PATH, BAILEYS_AUTH_DIR, STORAGE_LOCAL_PATH or+# PLUGINS_DIR). Preserve such a target separately, so a restore pointed at the wrong archive remains+# recoverable. OPENWA_RESTORE_SNAPSHOT_DIR takes these snapshots too when it is set: a target on its+# own mount under a read-only root has no writable place next to it.+snapshot_external() {target="$1"- resolved_target="$(resolve_path "$target")"- case "$resolved_target" in- "$RESOLVED_DATA_DIR"/*) ;;- *)- if [ -e "$target" ]; then- external_snapshot="${target}.pre-restore-$RESTORE_TIMESTAMP"- log "Snapshotting current $target -> $external_snapshot"- cp -p "$target" "$external_snapshot"- fi- ;;+ case "$(resolve_path "$target")" in+ "$RESOLVED_DATA_DIR"/*) return 0 ;;esac+ [ -e "$target" ] || return 0+ snapshot_dir="${OPENWA_RESTORE_SNAPSHOT_DIR:-$(dirname "$target")}"+ external_snapshot="${snapshot_dir%/}/$(basename "$target").pre-restore-$RESTORE_TIMESTAMP"+ # Targets from different parents can share a name in one snapshot dir (a PLUGINS_DIR and a+ # PLUGIN_STATE_DIR both end in plugins); never copy one into the other's snapshot.+ n=1+ while [ -e "$external_snapshot" ]; do+ n=$((n + 1))+ external_snapshot="${snapshot_dir%/}/$(basename "$target").pre-restore-$RESTORE_TIMESTAMP-$n"+ done+ log "Snapshotting current $target -> $external_snapshot"+ mkdir -p "$snapshot_dir"+ cp -pR "$target" "$external_snapshot"+}++# restore_db <staged file> <target> <label>+restore_db() {+ if [ "$PHASE" = snapshot ]; then+ snapshot_external "$2"+ return+ fi+ log "Restoring $3 -> $2"+ mkdir -p "$(dirname "$2")"+ cp "$1" "$2"+ # Owner-only, matching what the app re-tightens on every boot (sqlite-file-permissions.ts);+ # cp preserves the staged mode, and a foreign-umask extraction may leave it broader.+ chmod 0600 "$2" 2>/dev/null || true}# A restore target that already holds a working install's tables is LIVE: overwriting it destroys@@ -289,58 +303,56 @@ fimkdir -p "$DATA_DIR"-if [ -f "$STAGE/main.sqlite" ]; then- log "Restoring auth/audit DB -> $MAIN_DB"- snapshot_external_db "$MAIN_DB"- mkdir -p "$(dirname "$MAIN_DB")"- cp "$STAGE/main.sqlite" "$MAIN_DB"- # Owner-only, matching what the app re-tightens on every boot (sqlite-file-permissions.ts);- # cp preserves the staged mode, and a foreign-umask extraction may leave it broader.- chmod 0600 "$MAIN_DB" 2>/dev/null || true-else+if [ ! -f "$STAGE/main.sqlite" ]; thenlog "WARN: main.sqlite not in archive — API keys / audit log will NOT be restored"fi-if [ -f "$STAGE/openwa.sqlite" ]; then- log "Restoring data store -> $DATA_DB"- snapshot_external_db "$DATA_DB"- mkdir -p "$(dirname "$DATA_DB")"- cp "$STAGE/openwa.sqlite" "$DATA_DB"- chmod 0600 "$DATA_DB" 2>/dev/null || true+MERGED_PLUGINS_DIR=""+if [ -d "$STAGE/plugin-packages" ] && [ -d "$STAGE/plugin-state" ] &&+ [ "$(resolve_path "$PLUGIN_PACKAGES_DIR")" = "$(resolve_path "$PLUGIN_STATE_DIR")" ]; then+ # Docker deployments deliberately colocate package and state files. Build the complete target in+ # staging and replace it once, so neither half can erase the other during restore.+ MERGED_PLUGINS_DIR="$STAGE/plugin-merged"+ mkdir -p "$MERGED_PLUGINS_DIR"+ cp -pR "$STAGE/plugin-packages/." "$MERGED_PLUGINS_DIR"+ cp -pR "$STAGE/plugin-state/." "$MERGED_PLUGINS_DIR"fi-if [ -d "$STAGE/sessions" ]; then- replace_tree "$STAGE/sessions" "$SESSIONS_DIR" "whatsapp-web.js sessions"-fi--if [ -d "$STAGE/baileys" ]; then- replace_tree "$STAGE/baileys" "$BAILEYS_DIR" "Baileys authentication state"-fi--if [ -d "$STAGE/media" ]; then- replace_tree "$STAGE/media" "$MEDIA_DIR" "local media"-fi--if [ -d "$STAGE/plugin-packages" ] && [ -d "$STAGE/plugin-state" ]; then- RESOLVED_PLUGIN_PACKAGES_DIR="$(resolve_path "$PLUGIN_PACKAGES_DIR")"- RESOLVED_PLUGIN_STATE_DIR="$(resolve_path "$PLUGIN_STATE_DIR")"- if [ "$RESOLVED_PLUGIN_PACKAGES_DIR" = "$RESOLVED_PLUGIN_STATE_DIR" ]; then- # Docker deployments deliberately colocate package and state files. Build the complete target in- # staging and replace it once, so neither half can erase the other during restore.- MERGED_PLUGINS_DIR="$STAGE/plugin-merged"- mkdir -p "$MERGED_PLUGINS_DIR"- cp -pR "$STAGE/plugin-packages/." "$MERGED_PLUGINS_DIR"- cp -pR "$STAGE/plugin-state/." "$MERGED_PLUGINS_DIR"+# Runs twice: PHASE=snapshot checks every target and snapshots the ones outside the data dir, then+# PHASE=apply writes them. A target that cannot be snapshotted or is refused stops the restore+# before the first database is written, instead of halfway through with a mixed install left behind.+restore_targets() {+ if [ -f "$STAGE/main.sqlite" ]; then+ restore_db "$STAGE/main.sqlite" "$MAIN_DB" "auth/audit DB"+ fi+ if [ -f "$STAGE/openwa.sqlite" ]; then+ restore_db "$STAGE/openwa.sqlite" "$DATA_DB" "data store"+ fi+ if [ -d "$STAGE/sessions" ]; then+ replace_tree "$STAGE/sessions" "$SESSIONS_DIR" "whatsapp-web.js sessions"+ fi+ if [ -d "$STAGE/baileys" ]; then+ replace_tree "$STAGE/baileys" "$BAILEYS_DIR" "Baileys authentication state"+ fi+ if [ -d "$STAGE/media" ]; then+ replace_tree "$STAGE/media" "$MEDIA_DIR" "local media"+ fi+ if [ -n "$MERGED_PLUGINS_DIR" ]; thenreplace_tree "$MERGED_PLUGINS_DIR" "$PLUGIN_PACKAGES_DIR" "installed plugins and plugin state"else- replace_tree "$STAGE/plugin-packages" "$PLUGIN_PACKAGES_DIR" "installed plugin packages"- replace_tree "$STAGE/plugin-state" "$PLUGIN_STATE_DIR" "plugin registry and persisted state"+ if [ -d "$STAGE/plugin-packages" ]; then+ replace_tree "$STAGE/plugin-packages" "$PLUGIN_PACKAGES_DIR" "installed plugin packages"+ fi+ if [ -d "$STAGE/plugin-state" ]; then+ replace_tree "$STAGE/plugin-state" "$PLUGIN_STATE_DIR" "plugin registry and persisted state"+ fifi-elif [ -d "$STAGE/plugin-packages" ]; then- replace_tree "$STAGE/plugin-packages" "$PLUGIN_PACKAGES_DIR" "installed plugin packages"-elif [ -d "$STAGE/plugin-state" ]; then- replace_tree "$STAGE/plugin-state" "$PLUGIN_STATE_DIR" "plugin registry and persisted state"-fi+}++PHASE=snapshot+restore_targets+PHASE=apply+restore_targetsif [ -f "$STAGE/.env.generated" ]; thenlog "Restoring dashboard-generated configuration"diff --git a/scripts/smoke-test-backup-restore.sh b/scripts/smoke-test-backup-restore.shindex 0242d9a12..3187e735c 100755--- a/scripts/smoke-test-backup-restore.sh+++ b/scripts/smoke-test-backup-restore.sh@@ -17,6 +17,8 @@# (k) an operator's sqlite3 rc file changes neither answer of the guard (skipped without sqlite3)# (l) an unwritable BACKUP_DIR fails before anything is staged (skipped as root)# (m) OPENWA_RESTORE_SNAPSHOT_DIR takes the data-dir snapshot off a read-only parent (skipped as root)+# (n) a state dir outside the data dir is snapshotted before any database is written, and under+# OPENWA_RESTORE_SNAPSHOT_DIR when that is set (skipped as root)## Usage: ./scripts/smoke-test-backup-restore.sh# Requires: bash, tar, node (restore.sh path resolution). sqlite3 is optional (see (c) and (k)).@@ -531,5 +533,68 @@ elseecho "SKIP: (m) running as root, which ignores the permission bits this case relies on"fi+echo ""+echo "==> (n) a state dir outside the data dir is snapshotted before any database is written"+# SESSION_DATA_PATH on its own mount has a parent of its own. When that parent is read-only, the+# snapshot of the directory cannot go next to it, and a restore that finds out only after writing+# the databases leaves them from the archive and the sessions from the live install.+if [ "$(id -u)" -ne 0 ]; then+ N="$WORK/n"+ mkdir -p "$N/src/data/sessions/session-s1" "$N/live" "$N/ro/sessions/session-s1" "$N/ext/sessions/session-s1"+ make_fixture "$N/src/data/main.sqlite" "november-archive-main"+ make_fixture "$N/src/data/openwa.sqlite" "november-archive-data"+ printf 'november-archive\n' >"$N/src/data/sessions/session-s1/marker"+ (+ cd "$N/src"+ BACKUP_DIR="$N/out" "$BACKUP" >/dev/null+ )+ ARCHIVE_N="$(ls "$N"/out/openwa-backup-*.tar.gz)"+ make_fixture "$N/live/main.sqlite" "november-live-main"+ make_fixture "$N/live/openwa.sqlite" "november-live-data"+ printf 'november-live\n' >"$N/ro/sessions/session-s1/marker"+ printf 'november-live\n' >"$N/ext/sessions/session-s1/marker"++ # restore_n <sessions dir> [snapshot dir]: a forced restore of ARCHIVE_N over $N/live. Output lands+ # in OUT, the exit code in RC.+ restore_n() {+ set +e+ OUT="$(cd "$N" && MAIN_DATABASE_NAME="$N/live/main.sqlite" DATABASE_NAME="$N/live/openwa.sqlite" \+ OPENWA_DATA_DIR="$N/live" SESSION_DATA_PATH="$1" OPENWA_RESTORE_SNAPSHOT_DIR="${2:-}" \+ "$RESTORE" "$ARCHIVE_N" --force 2>&1)"+ RC=$?+ set -e+ }++ chmod a-w "$N/ro"+ restore_n "$N/ro/sessions"+ chmod u+w "$N/ro"+ if [ "$RC" -eq 0 ]; then+ fail "(n) restore exited 0 although the sessions snapshot could not be written"+ fi+ if [ "$(db_fingerprint "$N/live/main.sqlite")" != "november-live-main" ]; then+ fail "(n) the main DB was overwritten before the sessions snapshot failed"+ fi+ if [ "$(cat "$N/ro/sessions/session-s1/marker")" != "november-live" ]; then+ fail "(n) the failed restore changed the live sessions"+ fi++ restore_n "$N/ext/sessions" "$N/snapshots"+ if [ "$RC" -ne 0 ]; then+ fail "(n) restore with an external SESSION_DATA_PATH failed: $OUT"+ fi+ if [ "$(cat "$N"/snapshots/sessions.pre-restore-*/session-s1/marker 2>/dev/null || true)" != "november-live" ]; then+ fail "(n) the sessions snapshot is not under OPENWA_RESTORE_SNAPSHOT_DIR"+ fi+ if [ -n "$(ls -d "$N"/ext/sessions.pre-restore-* 2>/dev/null || true)" ]; then+ fail "(n) the sessions snapshot was written next to the target despite OPENWA_RESTORE_SNAPSHOT_DIR"+ fi+ if [ "$(cat "$N/ext/sessions/session-s1/marker")" != "november-archive" ]; then+ fail "(n) the archived sessions were not restored"+ fi+ pass "(n) external state snapshotted before the databases are written, under OPENWA_RESTORE_SNAPSHOT_DIR"+else+ echo "SKIP: (n) running as root, which ignores the permission bits this case relies on"+fi+echo ""echo "All smoke tests passed!"