🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Fix ClickHouse backup client timeout
Bill Yang99afb7ea5a2f
diff --git a/ops/backups/README.md b/ops/backups/README.mdindex 3f3de3d8c..11876556f 100644--- a/ops/backups/README.md+++ b/ops/backups/README.md@@ -15,6 +15,10 @@ PostgreSQL dumps include a custom-format database archive, cluster globals,and SHA-256 checksums. ClickHouse backs up only the `analytics` database, notthe large and regenerable `system` log tables.+The ClickHouse client receive timeout is raised to six hours because native+backups can produce no client-visible data for longer than the client's+five-minute default while the server is still making progress.+## PrerequisitesAll hosts need Bash, Docker, `flock`, `curl`, and `rclone`. The `clickhouse`diff --git a/ops/backups/bin/backup-clickhouse.sh b/ops/backups/bin/backup-clickhouse.shindex 2ef81d5c0..6c6d0addb 100755--- a/ops/backups/bin/backup-clickhouse.sh+++ b/ops/backups/bin/backup-clickhouse.sh@@ -31,11 +31,12 @@ run_clickhouse_query() {printf '%s\n' "$query" | docker exec -i \-e RYBBIT_BACKUP_USER="${CLICKHOUSE_BACKUP_USER:-}" \-e RYBBIT_BACKUP_PASSWORD="${CLICKHOUSE_BACKUP_PASSWORD:-}" \+ -e RYBBIT_BACKUP_RECEIVE_TIMEOUT="$CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT" \"$CLICKHOUSE_CONTAINER" sh -ec 'if [ -n "$RYBBIT_BACKUP_USER" ]; then- exec clickhouse-client --user "$RYBBIT_BACKUP_USER" --password "$RYBBIT_BACKUP_PASSWORD" --multiquery+ exec clickhouse-client --user "$RYBBIT_BACKUP_USER" --password "$RYBBIT_BACKUP_PASSWORD" --receive_timeout "$RYBBIT_BACKUP_RECEIVE_TIMEOUT" --multiqueryfi- exec clickhouse-client --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" --multiquery+ exec clickhouse-client --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" --receive_timeout "$RYBBIT_BACKUP_RECEIVE_TIMEOUT" --multiquery'}@@ -44,6 +45,7 @@ load_backup_configCLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-clickhouse}"CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-analytics}"CLICKHOUSE_BACKUP_MODE="${CLICKHOUSE_BACKUP_MODE:-s3}"+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT="${CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT:-21600}"CLICKHOUSE_LOCK_FILE="${CLICKHOUSE_LOCK_FILE:-/run/lock/rybbit-clickhouse-backup.lock}"BACKUP_HOST="${BACKUP_HOST:-$(hostname -s)}"DRY_RUN="${DRY_RUN:-false}"@@ -53,6 +55,8 @@ validate_simple_name CLICKHOUSE_CONTAINER "$CLICKHOUSE_CONTAINER"validate_identifier CLICKHOUSE_DATABASE "$CLICKHOUSE_DATABASE"[[ "$CLICKHOUSE_BACKUP_MODE" == "s3" || "$CLICKHOUSE_BACKUP_MODE" == "local" ]] || \fatal "CLICKHOUSE_BACKUP_MODE must be 's3' or 'local'"+[[ "$CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || \+ fatal "CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT must be a positive integer"BACKUP_NAME="${BACKUP_HOST}-clickhouse-${CLICKHOUSE_DATABASE}-$(date -u '+%Y%m%dT%H%M%SZ')"diff --git a/ops/backups/config/clickhouse.env.example b/ops/backups/config/clickhouse.env.exampleindex f3f876694..578c46a0b 100644--- a/ops/backups/config/clickhouse.env.example+++ b/ops/backups/config/clickhouse.env.example@@ -5,6 +5,7 @@ BACKUP_HOST=clickhouseCLICKHOUSE_CONTAINER=clickhouseCLICKHOUSE_DATABASE=analyticsCLICKHOUSE_BACKUP_MODE=local+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT=21600CLICKHOUSE_BACKUP_DISK=backupsCLICKHOUSE_LOCAL_BACKUP_ROOT=/var/backups/rybbit/clickhouseCLICKHOUSE_KEEP_LOCAL_BACKUP=falsediff --git a/ops/backups/config/frogstats.env.example b/ops/backups/config/frogstats.env.exampleindex 9f261d6a7..634b00c09 100644--- a/ops/backups/config/frogstats.env.example+++ b/ops/backups/config/frogstats.env.example@@ -12,6 +12,8 @@ POSTGRES_RCLONE_REMOTES=b2:rybbit-db-backups/frogstats/postgresCLICKHOUSE_CONTAINER=clickhouseCLICKHOUSE_DATABASE=analyticsCLICKHOUSE_BACKUP_MODE=s3+# Native backups can be quiet for longer than clickhouse-client's 300s default.+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT=21600CLICKHOUSE_BACKUP_USER=backup# Must match the SHA-256 hash installed in clickhouse-backup-user.xml.CLICKHOUSE_BACKUP_PASSWORD=replace-mediff --git a/ops/backups/tests/backup-clickhouse-timeout.sh b/ops/backups/tests/backup-clickhouse-timeout.shnew file mode 100755index 000000000..375c0a4b4--- /dev/null+++ b/ops/backups/tests/backup-clickhouse-timeout.sh@@ -0,0 +1,45 @@+#!/usr/bin/env bash++set -Eeuo pipefail++REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)}"+BACKUP_SCRIPT="${BACKUP_SCRIPT:-$REPO_ROOT/ops/backups/bin/backup-clickhouse.sh}"+TEST_ROOT="$(mktemp -d)"+trap 'find "$TEST_ROOT" -depth -delete' EXIT++mkdir -p "$TEST_ROOT/bin"++cat >"$TEST_ROOT/bin/docker" <<'EOF'+#!/usr/bin/env bash+printf '%s\n' "$*" >"$DOCKER_ARGS_LOG"+cat >/dev/null+EOF+chmod +x "$TEST_ROOT/bin/docker"++cat >"$TEST_ROOT/bin/flock" <<'EOF'+#!/usr/bin/env bash+exit 0+EOF+chmod +x "$TEST_ROOT/bin/flock"++cat >"$TEST_ROOT/config.env" <<'EOF'+BACKUP_HOST=test+CLICKHOUSE_CONTAINER=clickhouse+CLICKHOUSE_DATABASE=analytics+CLICKHOUSE_BACKUP_MODE=s3+CLICKHOUSE_BACKUP_USER=backup+CLICKHOUSE_BACKUP_PASSWORD=test-password+CLICKHOUSE_S3_NAMED_COLLECTION=b2_backups+CLICKHOUSE_BACKUP_RECEIVE_TIMEOUT=21600+EOF++export DOCKER_ARGS_LOG="$TEST_ROOT/docker-args"+PATH="$TEST_ROOT/bin:$PATH" \+ BACKUP_CONFIG="$TEST_ROOT/config.env" \+ CLICKHOUSE_LOCK_FILE="$TEST_ROOT/backup.lock" \+ "$BACKUP_SCRIPT" >/dev/null++grep -F -- '-e RYBBIT_BACKUP_RECEIVE_TIMEOUT=21600' "$DOCKER_ARGS_LOG" >/dev/null+grep -F -- '--receive_timeout "$RYBBIT_BACKUP_RECEIVE_TIMEOUT"' "$DOCKER_ARGS_LOG" >/dev/null++printf 'ClickHouse backup receive-timeout regression test passed\n'