Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Fix ClickHouse backup client timeout

Bill Yang99afb7ea5a2f
diff --git a/ops/backups/README.md b/ops/backups/README.md
index 3f3de3d8c..11876556f 100644
--- a/ops/backups/README.md
+++ b/ops/backups/README.md
@@ -15,6 +15,10 @@ PostgreSQL dumps include a custom-format database archive, cluster globals,
and SHA-256 checksums. ClickHouse backs up only the `analytics` database, not
the large and regenerable `system` log tables.
+The ClickHouse client receive timeout is raised to six hours because native
+backups can produce no client-visible data for longer than the client's
+five-minute default while the server is still making progress.
+
## Prerequisites
All hosts need Bash, Docker, `flock`, `curl`, and `rclone`. The `clickhouse`
diff --git a/ops/backups/bin/backup-clickhouse.sh b/ops/backups/bin/backup-clickhouse.sh
index 2ef81d5c0..6c6d0addb 100755
--- a/ops/backups/bin/backup-clickhouse.sh
+++ b/ops/backups/bin/backup-clickhouse.sh
@@ -31,11 +31,12 @@ run_clickhouse_query() {
printf '%s\n' "$query" | docker exec -i \
-e RYBBIT_BACKUP_USER="${CLICKHOUSE_BACKUP_USER:-}" \
-e RYBBIT_BACKUP_PASSWORD="${CLICKHOUSE_BACKUP_PASSWORD:-}" \
+ -e RYBBIT_BACKUP_RECEIVE_TIMEOUT="$CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT" \
"$CLICKHOUSE_CONTAINER" sh -ec '
if [ -n "$RYBBIT_BACKUP_USER" ]; then
- exec clickhouse-client --user "$RYBBIT_BACKUP_USER" --password "$RYBBIT_BACKUP_PASSWORD" --multiquery
+ exec clickhouse-client --user "$RYBBIT_BACKUP_USER" --password "$RYBBIT_BACKUP_PASSWORD" --receive_timeout "$RYBBIT_BACKUP_RECEIVE_TIMEOUT" --multiquery
fi
- exec clickhouse-client --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" --multiquery
+ exec clickhouse-client --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" --receive_timeout "$RYBBIT_BACKUP_RECEIVE_TIMEOUT" --multiquery
'
}
@@ -44,6 +45,7 @@ load_backup_config
CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-clickhouse}"
CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-analytics}"
CLICKHOUSE_BACKUP_MODE="${CLICKHOUSE_BACKUP_MODE:-s3}"
+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT="${CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT:-21600}"
CLICKHOUSE_LOCK_FILE="${CLICKHOUSE_LOCK_FILE:-/run/lock/rybbit-clickhouse-backup.lock}"
BACKUP_HOST="${BACKUP_HOST:-$(hostname -s)}"
DRY_RUN="${DRY_RUN:-false}"
@@ -53,6 +55,8 @@ validate_simple_name CLICKHOUSE_CONTAINER "$CLICKHOUSE_CONTAINER"
validate_identifier CLICKHOUSE_DATABASE "$CLICKHOUSE_DATABASE"
[[ "$CLICKHOUSE_BACKUP_MODE" == "s3" || "$CLICKHOUSE_BACKUP_MODE" == "local" ]] || \
fatal "CLICKHOUSE_BACKUP_MODE must be 's3' or 'local'"
+[[ "$CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || \
+ fatal "CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT must be a positive integer"
BACKUP_NAME="${BACKUP_HOST}-clickhouse-${CLICKHOUSE_DATABASE}-$(date -u '+%Y%m%dT%H%M%SZ')"
diff --git a/ops/backups/config/clickhouse.env.example b/ops/backups/config/clickhouse.env.example
index f3f876694..578c46a0b 100644
--- a/ops/backups/config/clickhouse.env.example
+++ b/ops/backups/config/clickhouse.env.example
@@ -5,6 +5,7 @@ BACKUP_HOST=clickhouse
CLICKHOUSE_CONTAINER=clickhouse
CLICKHOUSE_DATABASE=analytics
CLICKHOUSE_BACKUP_MODE=local
+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT=21600
CLICKHOUSE_BACKUP_DISK=backups
CLICKHOUSE_LOCAL_BACKUP_ROOT=/var/backups/rybbit/clickhouse
CLICKHOUSE_KEEP_LOCAL_BACKUP=false
diff --git a/ops/backups/config/frogstats.env.example b/ops/backups/config/frogstats.env.example
index 9f261d6a7..634b00c09 100644
--- a/ops/backups/config/frogstats.env.example
+++ b/ops/backups/config/frogstats.env.example
@@ -12,6 +12,8 @@ POSTGRES_RCLONE_REMOTES=b2:rybbit-db-backups/frogstats/postgres
CLICKHOUSE_CONTAINER=clickhouse
CLICKHOUSE_DATABASE=analytics
CLICKHOUSE_BACKUP_MODE=s3
+# Native backups can be quiet for longer than clickhouse-client's 300s default.
+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT=21600
CLICKHOUSE_BACKUP_USER=backup
# Must match the SHA-256 hash installed in clickhouse-backup-user.xml.
CLICKHOUSE_BACKUP_PASSWORD=replace-me
diff --git a/ops/backups/tests/backup-clickhouse-timeout.sh b/ops/backups/tests/backup-clickhouse-timeout.sh
new file mode 100755
index 000000000..375c0a4b4
--- /dev/null
+++ b/ops/backups/tests/backup-clickhouse-timeout.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)}"
+BACKUP_SCRIPT="${BACKUP_SCRIPT:-$REPO_ROOT/ops/backups/bin/backup-clickhouse.sh}"
+TEST_ROOT="$(mktemp -d)"
+trap 'find "$TEST_ROOT" -depth -delete' EXIT
+
+mkdir -p "$TEST_ROOT/bin"
+
+cat >"$TEST_ROOT/bin/docker" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$*" >"$DOCKER_ARGS_LOG"
+cat >/dev/null
+EOF
+chmod +x "$TEST_ROOT/bin/docker"
+
+cat >"$TEST_ROOT/bin/flock" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+chmod +x "$TEST_ROOT/bin/flock"
+
+cat >"$TEST_ROOT/config.env" <<'EOF'
+BACKUP_HOST=test
+CLICKHOUSE_CONTAINER=clickhouse
+CLICKHOUSE_DATABASE=analytics
+CLICKHOUSE_BACKUP_MODE=s3
+CLICKHOUSE_BACKUP_USER=backup
+CLICKHOUSE_BACKUP_PASSWORD=test-password
+CLICKHOUSE_S3_NAMED_COLLECTION=b2_backups
+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT=21600
+EOF
+
+export DOCKER_ARGS_LOG="$TEST_ROOT/docker-args"
+PATH="$TEST_ROOT/bin:$PATH" \
+ BACKUP_CONFIG="$TEST_ROOT/config.env" \
+ CLICKHOUSE_LOCK_FILE="$TEST_ROOT/backup.lock" \
+ "$BACKUP_SCRIPT" >/dev/null
+
+grep -F -- '-e RYBBIT_BACKUP_RECEIVE_TIMEOUT=21600' "$DOCKER_ARGS_LOG" >/dev/null
+grep -F -- '--receive_timeout "$RYBBIT_BACKUP_RECEIVE_TIMEOUT"' "$DOCKER_ARGS_LOG" >/dev/null
+
+printf 'ClickHouse backup receive-timeout regression test passed\n'