PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
set shell := ["bash", "-euo", "pipefail", "-c"]
lint:
golangci-lint fmt --diff ./cmd/... ./internal/...
golangci-lint run ./cmd/... ./internal/...
format:
golangci-lint fmt ./cmd/... ./internal/...
# The libedit shim lets the bundled PostgreSQL clients start on Fedora; without it a package that
# checks them at startup exits before its first test.
# Run the given test packages, e.g. just test-package ./internal/features/intervals/...
test-package +packages:
#!/usr/bin/env bash
set -euo pipefail
source ../scripts/fedora.sh
apply_libedit_shim
go test -count=1 {{ packages }}
# CI runs one group per runner, so the groups must cover every package exactly once. The other
# group drops only the subtrees the named groups take, so a new package lands in it. A group named
# <name>-<shard>-of-<count> runs one shard of <name>'s packages, see test-group-tests.
test-group-packages group:
#!/usr/bin/env bash
set -euo pipefail
group_name=$(sed -E 's/-[0-9]+-of-[0-9]+$//' <<<"{{ group }}")
case "$group_name" in
physical) go list ./internal/features/tests/physical/... ;;
logical) go list ./internal/features/tests/logical/... ;;
wal-usecases) go list ./internal/features/backups/backups/usecases/physical/postgresql ;;
other)
go list ./internal/... |
grep -Ev '/internal/features/(tests/(physical|logical)(/|$)|backups/backups/usecases/physical/postgresql$)'
;;
*) echo "unknown test group: {{ group }} (expected physical, logical, wal-usecases or other)" >&2; exit 1 ;;
esac
# A single slow package bounds its group however the packages are spread, so a sharded group splits
# the top-level tests of its packages instead. Each test goes to exactly one shard, the least-loaded
# one by the weights in test-shard-weights.txt. Prints a -run pattern, or nothing for a group that
# is not sharded.
test-group-tests group:
#!/usr/bin/env bash
set -euo pipefail
if [[ ! "{{ group }}" =~ -([0-9]+)-of-([0-9]+)$ ]]; then
exit 0
fi
shard=${BASH_REMATCH[1]}
shard_count=${BASH_REMATCH[2]}
if ((shard < 1 || shard > shard_count)); then
echo "shard ${shard} is outside 1..${shard_count} in test group {{ group }}" >&2
exit 1
fi
packages=$(just test-group-packages "{{ group }}")
# go test -list would run each package's TestMain, which starts its database containers, so
# the names come from the test sources.
# shellcheck disable=SC2086 # one import path per word
package_directories=$(go list -f '{{{{ .Dir }}' $packages)
shopt -s nullglob
test_files=()
for package_directory in $package_directories; do
test_files+=("$package_directory"/*_test.go)
done
test_names=$(
grep -hoE '^func (Test|Example|Fuzz)[A-Za-z0-9_]*' "${test_files[@]}" | sed 's/^func //' | grep -vx 'TestMain' | sort -u
)
shard_names=$(
awk 'NR == FNR { if ($0 !~ /^#/ && NF == 2) weights[$2] = $1; next } { print ($1 in weights ? weights[$1] : 1), $1 }' \
test-shard-weights.txt - <<<"$test_names" |
sort -k1,1nr -k2,2 |
awk -v shard="$shard" -v count="$shard_count" '{
lightest = 1
for (candidate = 2; candidate <= count; candidate++) if (load[candidate] < load[lightest]) lightest = candidate
load[lightest] += $1
if (lightest == shard) print $2
}'
)
if [[ -z "$shard_names" ]]; then
echo "shard ${shard} of ${shard_count} in test group {{ group }} has no tests" >&2
exit 1
fi
echo "^($(paste -sd'|' <<<"$shard_names"))\$"
clean-testcontainers:
#!/usr/bin/env bash
set -euo pipefail
ids=$(docker ps -aq --filter "label=org.testcontainers")
if [[ -n "$ids" ]]; then docker rm -f $ids; else echo "no leftover testcontainers"; fi
# For the bare names the tag must start with a digit, so a quoted "postgres:postgres" chown arg in
# physical tests isn't mistaken for an image tag; every real image tag there is version-like. A
# namespaced name (owner/image) cannot be confused with anything, so its tag is matched as written.
pull-testcontainers:
grep -rhoE '"(mysql|mariadb|mongo|postgres):[0-9][a-zA-Z0-9._-]*"|"pgsty/minio:[a-zA-Z0-9._-]+"|"nicolaka/netshoot:[a-zA-Z0-9._-]+"' ./internal \
| tr -d '"' | sort -u \
| xargs -P 4 -I {} sh -c 'for n in 1 2 3; do docker pull {} && exit 0; echo "retry {} ($n)"; sleep 3; done; echo "FAILED to pull {} after 3 attempts"; exit 1'