Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

4.5 KB
Raw Blame
set shell := ["bash", "-euo", "pipefail", "-c"]

lint:
    golangci-lint fmt --diff ./cmd/... ./internal/...
    golangci-lint run ./cmd/... ./internal/...

format:
    golangci-lint fmt ./cmd/... ./internal/...

# The libedit shim lets the bundled PostgreSQL clients start on Fedora; without it a package that
# checks them at startup exits before its first test.
# Run the given test packages, e.g. just test-package ./internal/features/intervals/...
test-package +packages:
    #!/usr/bin/env bash
    set -euo pipefail
    source ../scripts/fedora.sh
    apply_libedit_shim
    go test -count=1 {{ packages }}

# CI runs one group per runner, so the groups must cover every package exactly once. The other
# group drops only the subtrees the named groups take, so a new package lands in it. A group named
# <name>-<shard>-of-<count> runs one shard of <name>'s packages, see test-group-tests.
test-group-packages group:
    #!/usr/bin/env bash
    set -euo pipefail
    group_name=$(sed -E 's/-[0-9]+-of-[0-9]+$//' <<<"{{ group }}")
    case "$group_name" in
        physical) go list ./internal/features/tests/physical/... ;;
        logical) go list ./internal/features/tests/logical/... ;;
        wal-usecases) go list ./internal/features/backups/backups/usecases/physical/postgresql ;;
        other)
            go list ./internal/... |
                grep -Ev '/internal/features/(tests/(physical|logical)(/|$)|backups/backups/usecases/physical/postgresql$)'
            ;;
        *) echo "unknown test group: {{ group }} (expected physical, logical, wal-usecases or other)" >&2; exit 1 ;;
    esac

# A single slow package bounds its group however the packages are spread, so a sharded group splits
# the top-level tests of its packages instead. Each test goes to exactly one shard, the least-loaded
# one by the weights in test-shard-weights.txt. Prints a -run pattern, or nothing for a group that
# is not sharded.
test-group-tests group:
    #!/usr/bin/env bash
    set -euo pipefail
    if [[ ! "{{ group }}" =~ -([0-9]+)-of-([0-9]+)$ ]]; then
        exit 0
    fi
    shard=${BASH_REMATCH[1]}
    shard_count=${BASH_REMATCH[2]}
    if ((shard < 1 || shard > shard_count)); then
        echo "shard ${shard} is outside 1..${shard_count} in test group {{ group }}" >&2
        exit 1
    fi
    packages=$(just test-group-packages "{{ group }}")
    # go test -list would run each package's TestMain, which starts its database containers, so
    # the names come from the test sources.
    # shellcheck disable=SC2086 # one import path per word
    package_directories=$(go list -f '{{{{ .Dir }}' $packages)
    shopt -s nullglob
    test_files=()
    for package_directory in $package_directories; do
        test_files+=("$package_directory"/*_test.go)
    done
    test_names=$(
        grep -hoE '^func (Test|Example|Fuzz)[A-Za-z0-9_]*' "${test_files[@]}" | sed 's/^func //' | grep -vx 'TestMain' | sort -u
    )
    shard_names=$(
        awk 'NR == FNR { if ($0 !~ /^#/ && NF == 2) weights[$2] = $1; next } { print ($1 in weights ? weights[$1] : 1), $1 }' \
            test-shard-weights.txt - <<<"$test_names" |
            sort -k1,1nr -k2,2 |
            awk -v shard="$shard" -v count="$shard_count" '{
                lightest = 1
                for (candidate = 2; candidate <= count; candidate++) if (load[candidate] < load[lightest]) lightest = candidate
                load[lightest] += $1
                if (lightest == shard) print $2
            }'
    )
    if [[ -z "$shard_names" ]]; then
        echo "shard ${shard} of ${shard_count} in test group {{ group }} has no tests" >&2
        exit 1
    fi
    echo "^($(paste -sd'|' <<<"$shard_names"))\$"

clean-testcontainers:
    #!/usr/bin/env bash
    set -euo pipefail
    ids=$(docker ps -aq --filter "label=org.testcontainers")
    if [[ -n "$ids" ]]; then docker rm -f $ids; else echo "no leftover testcontainers"; fi

# For the bare names the tag must start with a digit, so a quoted "postgres:postgres" chown arg in
# physical tests isn't mistaken for an image tag; every real image tag there is version-like. A
# namespaced name (owner/image) cannot be confused with anything, so its tag is matched as written.
pull-testcontainers:
    grep -rhoE '"(mysql|mariadb|mongo|postgres):[0-9][a-zA-Z0-9._-]*"|"pgsty/minio:[a-zA-Z0-9._-]+"|"nicolaka/netshoot:[a-zA-Z0-9._-]+"' ./internal \
        | tr -d '"' | sort -u \
        | xargs -P 4 -I {} sh -c 'for n in 1 2 3; do docker pull {} && exit 0; echo "retry {} ($n)"; sleep 3; done; echo "FAILED to pull {} after 3 attempts"; exit 1'