PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
#!/usr/bin/env bash
set -Eeuo pipefail
readonly harness_label_key="com.databasus.storage-test"
run_id="${RANDOM}-$$-$(date +%s)"
readonly run_id
readonly harness_label="${harness_label_key}=${run_id}"
readonly selected_case="${CASE:-all}"
readonly databasus_image="${DATABASUS_IMAGE:?DATABASUS_IMAGE must be set}"
readonly samba_fixture_image="dockurr/samba@sha256:dbd8ebd6392cc1e8e98312d96d163f0039b644b84aee741d89efdd0abd3d32bf"
readonly nfs_fixture_image="itsthenetwork/nfs-server-alpine@sha256:7fa99ae65c23c5af87dd4300e543a86b119ed15ba61422444207efc7abd0ba20"
suite_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly suite_directory
case_root=""
cleanup() {
local container_id
local container_name
local volume_name
local volume_type
local -a client_container_ids=()
local -a server_container_ids=()
local -a volume_names=()
local -a network_ids=()
while read -r container_id container_name; do
[[ -n "${container_id}" ]] || continue
if [[ "${container_name}" == *-server-* ]]; then
server_container_ids+=("${container_id}")
else
client_container_ids+=("${container_id}")
fi
done < <(docker ps -a --filter "label=${harness_label}" --format '{{.ID}} {{.Names}}' 2>/dev/null)
if (("${#client_container_ids[@]}")); then
docker rm -fv "${client_container_ids[@]}" >/dev/null 2>&1 || true
fi
mapfile -t volume_names < <(docker volume ls -q --filter "label=${harness_label}" 2>/dev/null)
for volume_name in "${volume_names[@]}"; do
volume_type="$(docker volume inspect --format '{{index .Options "type"}}' "${volume_name}" 2>/dev/null || true)"
if [[ "${volume_type}" == "cifs" || "${volume_type}" == "nfs" ]]; then
docker volume rm -f "${volume_name}" >/dev/null 2>&1 || true
fi
done
if (("${#server_container_ids[@]}")); then
docker rm -fv "${server_container_ids[@]}" >/dev/null 2>&1 || true
fi
mapfile -t network_ids < <(docker network ls -q --filter "label=${harness_label}" 2>/dev/null)
if (("${#network_ids[@]}")); then
docker network rm "${network_ids[@]}" >/dev/null 2>&1 || true
fi
mapfile -t volume_names < <(docker volume ls -q --filter "label=${harness_label}" 2>/dev/null)
if (("${#volume_names[@]}")); then
docker volume rm -f "${volume_names[@]}" >/dev/null 2>&1 || true
fi
if [[ "${case_root}" == /tmp/databasus-storage-test.* ]]; then
docker run --rm --label "${harness_label}" --entrypoint /bin/chmod \
--volume "${case_root}:/case-root" "${databasus_image}" \
-R 0777 /case-root >/dev/null 2>&1 || true
rm -rf -- "${case_root}" || true
fi
}
trap cleanup EXIT
case_root="$(mktemp -d /tmp/databasus-storage-test.XXXXXXXX)"
wait_for_health() {
local container_name="$1"
local health_status
for ((readiness_attempt = 1; readiness_attempt <= 120; readiness_attempt++)); do
health_status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "${container_name}")"
if [[ "${health_status}" == "healthy" ]]; then
return
fi
if [[ "$(docker inspect --format '{{.State.Running}}' "${container_name}")" != "true" ]]; then
docker logs "${container_name}" >&2
return 1
fi
sleep 1
done
docker logs "${container_name}" >&2
echo "container did not become healthy" >&2
return 1
}
wait_for_log() {
local container_name="$1"
local required_log_line="$2"
for ((log_attempt = 1; log_attempt <= 60; log_attempt++)); do
if docker logs "${container_name}" 2>&1 | grep -F "${required_log_line}" >/dev/null; then
return
fi
sleep 1
done
docker logs "${container_name}" >&2
echo "container did not report readiness: ${required_log_line}" >&2
return 1
}
wait_for_stop() {
local container_name="$1"
for ((exit_attempt = 1; exit_attempt <= 100; exit_attempt++)); do
if [[ "$(docker inspect --format '{{.State.Running}}' "${container_name}")" != "true" ]]; then
return
fi
sleep 0.1
done
echo "container did not stop" >&2
return 1
}
get_exit_code() {
local container_name="$1"
docker inspect --format '{{.State.ExitCode}}' "${container_name}"
}
wait_for_exit() {
local container_name="$1"
wait_for_stop "${container_name}"
[[ "$(get_exit_code "${container_name}")" != "0" ]]
}
start_databasus() {
local container_name="$1"
shift
docker run --detach \
--name "${container_name}" \
--label "${harness_label}" \
--health-interval 1s \
--health-start-period 1s \
"$@" \
"${databasus_image}" >/dev/null
wait_for_health "${container_name}"
}
start_failing_databasus() {
local container_name="$1"
shift
docker run --detach \
--name "${container_name}" \
--label "${harness_label}" \
"$@" \
"${databasus_image}" >/dev/null
wait_for_exit "${container_name}"
}
assert_storage_permission_failure() {
local container_name="$1"
local container_logs
container_logs="$(docker logs "${container_name}" 2>&1)"
grep -Eq 'ERROR: Databasus cannot write to .+ as UID [0-9]+ and GID [0-9]+\.' <<<"${container_logs}"
grep -Fq "Required operation:" <<<"${container_logs}"
grep -Fq "Set PUID and PGID or fix the mounted directory permissions:" <<<"${container_logs}"
grep -Fq "https://databasus.com/advanced-config/#docker-storage-permissions" <<<"${container_logs}"
if grep -Fq "Starting PostgreSQL..." <<<"${container_logs}"; then
echo "PostgreSQL started after a failed storage check" >&2
return 1
fi
if grep -Fq "Starting Databasus application..." <<<"${container_logs}"; then
echo "Databasus started after a failed storage check" >&2
return 1
fi
assert_no_storage_probe_remains "${container_name}"
}
assert_no_storage_probe_remains() {
local container_name="$1"
local remaining_probe
remaining_probe="$(docker run --rm \
--label "${harness_label}" \
--volumes-from "${container_name}" \
--entrypoint /bin/sh \
"${databasus_image}" \
-c 'find /databasus-data/temp /databasus-data/backups \
-type f -name ".databasus-storage-test-*" -print -quit 2>/dev/null || true')"
if [ -n "${remaining_probe}" ]; then
echo "Storage probe remains after failed startup: ${remaining_probe}" >&2
return 1
fi
}
assert_storage_command_saves_and_deletes_probe() {
local container_name="$1"
docker exec --user databasus "${container_name}" \
databasus --test-storage 2>&1 | grep -Fx "storage test passed" >/dev/null
}
assert_single_runtime_identity() {
local container_name="$1"
local runtime_gid
local runtime_uid
local service_pid
runtime_uid="$(docker exec "${container_name}" id -u databasus)"
runtime_gid="$(docker exec "${container_name}" id -g databasus)"
[[ "${runtime_uid}" != "0" ]]
[[ "${runtime_gid}" != "0" ]]
if docker exec "${container_name}" id postgres >/dev/null 2>&1; then
echo "The image still contains a postgres operating-system account" >&2
return 1
fi
[[ "$(docker exec "${container_name}" stat -c %u /proc/1)" == "${runtime_uid}" ]]
[[ "$(docker exec "${container_name}" stat -c %g /proc/1)" == "${runtime_gid}" ]]
service_pid="$(docker exec "${container_name}" /bin/sh -eu -c '
pidof postgres | cut -d" " -f1
')"
[[ "$(docker exec "${container_name}" stat -c %u "/proc/${service_pid}")" == "${runtime_uid}" ]]
[[ "$(docker exec "${container_name}" stat -c %g "/proc/${service_pid}")" == "${runtime_gid}" ]]
}
get_generated_database_password() {
local container_name="$1"
docker exec "${container_name}" \
sed -n 's/.* password=\([^ ]*\) .*/\1/p' /dev/shm/databasus-database-dsn
}
can_log_in_over_loopback() {
local container_name="$1"
local database_password="$2"
docker exec -i --user databasus "${container_name}" /bin/sh -c '
IFS= read -r PGPASSWORD
export PGPASSWORD
/usr/lib/postgresql/17/bin/psql -h localhost -p 5437 -U postgres -d postgres -c "SELECT 1"
' <<<"${database_password}" >/dev/null 2>&1
}
assert_postgresql_runtime_authentication() {
local container_name="$1"
local database_password
local replication_probe_output
[[ "$(docker exec "${container_name}" stat -c %a /databasus-data/pgdata/pg_hba.conf)" == "600" ]]
[[ "$(docker exec "${container_name}" stat -c %a /databasus-data/pgdata/pg_ident.conf)" == "600" ]]
if docker exec "${container_name}" grep -Fq ".s.PGSQL" /proc/net/unix; then
echo "PostgreSQL created a Unix socket" >&2
return 1
fi
if docker exec --user databasus "${container_name}" \
env -u PGPASSWORD /usr/lib/postgresql/17/bin/psql \
-h localhost -p 5437 -U postgres -d postgres \
-c 'SELECT 1' >/dev/null 2>&1; then
echo "PostgreSQL accepted a loopback login without its generated password" >&2
return 1
fi
database_password="$(get_generated_database_password "${container_name}")"
[[ -n "${database_password}" ]]
if docker logs "${container_name}" 2>&1 | grep -F "${database_password}" >/dev/null; then
echo "The container log contains the generated PostgreSQL password" >&2
return 1
fi
if docker exec "${container_name}" /bin/sh -c 'cat /proc/[0-9]*/cmdline 2>/dev/null' |
tr '\0' ' ' | grep -F "${database_password}" >/dev/null; then
echo "A process in the container carries the generated PostgreSQL password in its arguments" >&2
return 1
fi
replication_probe_output="$(docker exec -i --user databasus "${container_name}" /bin/sh -c '
IFS= read -r PGPASSWORD
export PGPASSWORD
/usr/lib/postgresql/17/bin/psql \
"host=localhost port=5437 user=postgres dbname=postgres replication=database" \
-c IDENTIFY_SYSTEM
' <<<"${database_password}" 2>&1 || true)"
if ! grep -Fq "number of requested standby connections exceeds" \
<<<"${replication_probe_output}"; then
echo "PostgreSQL did not reject a replication connection: ${replication_probe_output}" >&2
return 1
fi
}
assert_running_container() {
local container_name="$1"
assert_storage_command_saves_and_deletes_probe "${container_name}"
assert_single_runtime_identity "${container_name}"
assert_postgresql_runtime_authentication "${container_name}"
}
prepare_bind_tree() {
local data_root="$1"
local data_uid="$2"
local data_gid="$3"
local backup_uid="${4:-${data_uid}}"
local backup_gid="${5:-${data_gid}}"
mkdir -p "${data_root}"
docker run --rm --label "${harness_label}" --entrypoint /bin/sh \
--volume "${data_root}:/data" "${databasus_image}" -eu -c '
mkdir -p /data/pgdata /data/temp /data/backups
chown "$1:$2" /data /data/pgdata /data/temp
chown "$3:$4" /data/backups
chmod 0770 /data /data/backups
chmod 0700 /data/pgdata /data/temp
' sh "${data_uid}" "${data_gid}" "${backup_uid}" "${backup_gid}"
}
run_default_ids() {
local container_name="databasus-storage-default-ids-${run_id}"
local volume_name="databasus-storage-default-ids-${run_id}"
local image_environment
image_environment="$(docker image inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "${databasus_image}")"
if grep -Eq '^(PUID|PGID|DATABASUS_PUID|DATABASUS_PGID|POSTGRES_PUID|POSTGRES_PGID)=' <<<"${image_environment}"; then
echo "The image publishes a runtime identity default" >&2
return 1
fi
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
start_databasus "${container_name}" --volume "${volume_name}:/databasus-data"
[[ "$(docker exec "${container_name}" id -u databasus)" == "999" ]]
[[ "$(docker exec "${container_name}" id -g databasus)" == "999" ]]
assert_running_container "${container_name}"
}
run_custom_ids() {
local container_name="databasus-storage-custom-ids-${run_id}"
local volume_name="databasus-storage-custom-ids-${run_id}"
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
start_databasus "${container_name}" \
--env PUID=25001 \
--env PGID=25002 \
--volume "${volume_name}:/databasus-data"
[[ "$(docker exec "${container_name}" id -u databasus)" == "25001" ]]
[[ "$(docker exec "${container_name}" id -g databasus)" == "25002" ]]
assert_running_container "${container_name}"
}
run_maximum_ids() {
local container_name="databasus-storage-maximum-ids-${run_id}"
local volume_name="databasus-storage-maximum-ids-${run_id}"
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
start_databasus "${container_name}" \
--env PUID=4294967294 \
--env PGID=4294967294 \
--volume "${volume_name}:/databasus-data"
[[ "$(docker exec "${container_name}" id -u databasus)" == "4294967294" ]]
[[ "$(docker exec "${container_name}" id -g databasus)" == "4294967294" ]]
assert_running_container "${container_name}"
}
run_automatic_ids() {
local container_name="databasus-storage-automatic-ids-${run_id}"
local data_root="${case_root}/automatic-ids"
prepare_bind_tree "${data_root}" 27501 27502
start_databasus "${container_name}" --volume "${data_root}:/databasus-data"
[[ "$(docker exec "${container_name}" id -u databasus)" == "27501" ]]
[[ "$(docker exec "${container_name}" id -g databasus)" == "27502" ]]
assert_running_container "${container_name}"
}
run_empty_pgdata_ids() {
local container_name="databasus-storage-empty-pgdata-ids-${run_id}"
local data_volume_name="databasus-storage-empty-pgdata-data-${run_id}"
local pgdata_volume_name="databasus-storage-empty-pgdata-${run_id}"
docker volume create --label "${harness_label}" "${data_volume_name}" >/dev/null
docker volume create --label "${harness_label}" "${pgdata_volume_name}" >/dev/null
docker run --rm --label "${harness_label}" --entrypoint /bin/chown \
--volume "${pgdata_volume_name}:/pgdata" "${databasus_image}" \
27501:27502 /pgdata
start_databasus "${container_name}" \
--volume "${data_volume_name}:/databasus-data" \
--volume "${pgdata_volume_name}:/databasus-data/pgdata"
[[ "$(docker exec "${container_name}" id -u databasus)" == "27501" ]]
[[ "$(docker exec "${container_name}" id -g databasus)" == "27502" ]]
assert_running_container "${container_name}"
}
run_partial_ids() {
local container_name="databasus-storage-partial-ids-${run_id}"
local data_root="${case_root}/partial-ids"
prepare_bind_tree "${data_root}" 27501 27502
start_databasus "${container_name}" \
--env PUID=25001 \
--volume "${data_root}:/databasus-data"
[[ "$(docker exec "${container_name}" id -u databasus)" == "25001" ]]
[[ "$(docker exec "${container_name}" id -g databasus)" == "27502" ]]
assert_running_container "${container_name}"
}
run_invalid_ids() {
local invalid_value
local container_name
for invalid_value in "" 0 abc 4294967295 18446744073709551617; do
container_name="databasus-storage-invalid-id-${run_id}-${RANDOM}"
start_failing_databasus "${container_name}" --env "PUID=${invalid_value}"
docker logs "${container_name}" 2>&1 | grep -F "ERROR: PUID must be a non-zero decimal Linux ID" >/dev/null
done
container_name="databasus-storage-colliding-id-${run_id}"
start_failing_databasus "${container_name}" --env PUID=1
docker logs "${container_name}" 2>&1 | grep -F "belongs to account" >/dev/null
}
run_bind_root() {
local container_name="databasus-storage-bind-${run_id}"
local data_root="${case_root}/bind-root"
prepare_bind_tree "${data_root}" 999 999
start_databasus "${container_name}" --volume "${data_root}:/databasus-data"
assert_running_container "${container_name}"
}
run_named_volume() {
local container_name="databasus-storage-volume-${run_id}"
local volume_name="databasus-storage-volume-${run_id}"
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
start_databasus "${container_name}" --volume "${volume_name}:/databasus-data"
assert_running_container "${container_name}"
}
run_split_mounts() {
local container_name="databasus-storage-split-${run_id}"
local data_volume_name="databasus-storage-split-data-${run_id}"
local backup_volume_name="databasus-storage-split-backup-${run_id}"
local pgdata_volume_name="databasus-storage-split-pgdata-${run_id}"
local temporary_device_id
local backup_device_id
docker volume create --label "${harness_label}" "${data_volume_name}" >/dev/null
docker volume create --label "${harness_label}" "${backup_volume_name}" >/dev/null
docker volume create --label "${harness_label}" "${pgdata_volume_name}" >/dev/null
start_databasus "${container_name}" \
--volume "${data_volume_name}:/databasus-data" \
--tmpfs /databasus-data/temp:rw,nosuid,size=64m,mode=0700,uid=999,gid=999 \
--volume "${backup_volume_name}:/databasus-data/backups" \
--volume "${pgdata_volume_name}:/databasus-data/pgdata"
temporary_device_id="$(docker exec "${container_name}" stat -c %d /databasus-data/temp)"
backup_device_id="$(docker exec "${container_name}" stat -c %d /databasus-data/backups)"
[[ "${temporary_device_id}" != "${backup_device_id}" ]]
assert_running_container "${container_name}"
}
run_no_unix_sockets() {
local container_name="databasus-storage-no-unix-sockets-${run_id}"
local volume_name="databasus-storage-no-unix-sockets-${run_id}"
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
start_databasus "${container_name}" \
--security-opt "seccomp=${suite_directory}/no-unix-sockets.json" \
--volume "${volume_name}:/databasus-data"
assert_running_container "${container_name}"
}
run_metadata_changes_denied() {
local container_name="databasus-storage-metadata-denied-${run_id}"
local data_root="${case_root}/metadata-denied"
prepare_bind_tree "${data_root}" 999 999
start_databasus "${container_name}" \
--cap-drop CHOWN \
--cap-drop FOWNER \
--env PUID=999 \
--env PGID=999 \
--volume "${data_root}:/databasus-data"
assert_running_container "${container_name}"
}
run_read_only() {
local container_name="databasus-storage-read-only-${run_id}"
local data_root="${case_root}/read-only"
prepare_bind_tree "${data_root}" 999 999
# The Docker engine on Namespace CI runners drops the readonly key of --mount
# and mounts the directory writable, while it honours the :ro suffix of --volume.
start_failing_databasus "${container_name}" \
--env PUID=999 \
--env PGID=999 \
--volume "${data_root}:/databasus-data:ro"
assert_storage_permission_failure "${container_name}"
}
run_wrong_owner() {
local container_name="databasus-storage-wrong-owner-${run_id}"
local data_root="${case_root}/wrong-owner"
prepare_bind_tree "${data_root}" 999 999 27501 27502
start_failing_databasus "${container_name}" \
--cap-drop CHOWN \
--cap-drop FOWNER \
--env PUID=999 \
--env PGID=999 \
--volume "${data_root}:/databasus-data"
assert_storage_permission_failure "${container_name}"
docker logs "${container_name}" 2>&1 | grep -F \
"Required operation: save a file through local storage." >/dev/null
}
run_unwritable_data_root() {
local container_name="databasus-storage-unwritable-root-${run_id}"
local data_root="${case_root}/unwritable-root"
prepare_bind_tree "${data_root}" 999 999
docker run --rm --label "${harness_label}" --entrypoint /bin/sh \
--volume "${data_root}:/data" "${databasus_image}" -eu -c '
chown 27501:27502 /data
chmod 0555 /data
'
start_failing_databasus "${container_name}" \
--cap-drop CHOWN \
--cap-drop DAC_OVERRIDE \
--cap-drop FOWNER \
--env PUID=999 \
--env PGID=999 \
--volume "${data_root}:/databasus-data"
assert_storage_permission_failure "${container_name}"
docker logs "${container_name}" 2>&1 | grep -F \
"Required operation: create, read, and remove a required file." >/dev/null
}
run_network_prerequisites() {
[[ "$(uname -s)" == "Linux" ]]
command -v mount.cifs >/dev/null || {
echo "mount.cifs is required; install cifs-utils on the Docker host" >&2
return 1
}
command -v mount.nfs >/dev/null || {
echo "mount.nfs is required; install nfs-common on the Docker host" >&2
return 1
}
docker image inspect "${samba_fixture_image}" >/dev/null 2>&1 || docker pull "${samba_fixture_image}" >/dev/null
docker image inspect "${nfs_fixture_image}" >/dev/null 2>&1 || docker pull "${nfs_fixture_image}" >/dev/null
}
run_cifs() {
local network_name="databasus-storage-cifs-${run_id}"
local server_container_name="databasus-storage-cifs-server-${run_id}"
local client_container_name="databasus-storage-cifs-client-${run_id}"
local server_volume_name="databasus-storage-cifs-server-${run_id}"
local data_volume_name="databasus-storage-cifs-data-${run_id}"
local backup_volume_name="databasus-storage-cifs-backup-${run_id}"
local server_address
run_network_prerequisites
docker network create --label "${harness_label}" "${network_name}" >/dev/null
docker volume create --label "${harness_label}" "${server_volume_name}" >/dev/null
docker volume create --label "${harness_label}" "${data_volume_name}" >/dev/null
docker run --rm --label "${harness_label}" --entrypoint /bin/sh \
--volume "${server_volume_name}:/shared" "${databasus_image}" -c \
'chown 999:999 /shared && chmod 0770 /shared'
docker run --detach \
--name "${server_container_name}" \
--label "${harness_label}" \
--network "${network_name}" \
--health-interval 1s \
--env NAME=Storage \
--env USER=storage-test \
--env PASS=storage-test-password \
--env UID=999 \
--env GID=999 \
--volume "${server_volume_name}:/shared" \
"${samba_fixture_image}" >/dev/null
wait_for_health "${server_container_name}"
server_address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${server_container_name}")"
docker volume create \
--label "${harness_label}" \
--driver local \
--opt type=cifs \
--opt "device=//${server_address}/Storage" \
--opt "o=username=storage-test,password=storage-test-password,vers=3.0,uid=0,gid=999,file_mode=0660,dir_mode=0770" \
"${backup_volume_name}" >/dev/null
start_databasus "${client_container_name}" \
--volume "${data_volume_name}:/databasus-data" \
--volume "${backup_volume_name}:/databasus-data/backups"
[[ "$(docker exec "${client_container_name}" stat -c %u /databasus-data/backups)" == "0" ]]
[[ "$(docker exec "${client_container_name}" stat -c %g /databasus-data/backups)" == "999" ]]
[[ "$(docker exec "${client_container_name}" stat -c %a /databasus-data/backups)" == "770" ]]
docker exec --user databasus "${client_container_name}" /bin/sh -eu -c '
printf fixed-mode > /databasus-data/backups/.fixed-mode-check
test "$(stat -c %a /databasus-data/backups/.fixed-mode-check)" = 660
rm /databasus-data/backups/.fixed-mode-check
'
assert_running_container "${client_container_name}"
}
run_nfs_root_squash() {
local network_name="databasus-storage-nfs-${run_id}"
local server_container_name="databasus-storage-nfs-server-${run_id}"
local client_container_name="databasus-storage-nfs-client-${run_id}"
local server_volume_name="databasus-storage-nfs-server-${run_id}"
local pgdata_volume_name="databasus-storage-nfs-pgdata-${run_id}"
local client_volume_name="databasus-storage-nfs-client-${run_id}"
local server_address
run_network_prerequisites
docker network create --label "${harness_label}" "${network_name}" >/dev/null
docker volume create --label "${harness_label}" "${server_volume_name}" >/dev/null
docker volume create --label "${harness_label}" "${pgdata_volume_name}" >/dev/null
docker run --rm --label "${harness_label}" --entrypoint /bin/sh \
--volume "${server_volume_name}:/nfsshare" "${databasus_image}" -c \
'mkdir -p /nfsshare/pgdata /nfsshare/temp &&
chown -R 999:999 /nfsshare &&
chmod 0755 /nfsshare /nfsshare/pgdata /nfsshare/temp'
docker run --detach \
--name "${server_container_name}" \
--label "${harness_label}" \
--network "${network_name}" \
--privileged \
--env SHARED_DIRECTORY=/nfsshare \
--volume "${server_volume_name}:/nfsshare" \
--entrypoint /bin/bash \
"${nfs_fixture_image}" -c \
"mount -t nfsd nfsd /proc/fs/nfsd && printf 10 > /proc/fs/nfsd/nfsv4gracetime && sed -i 's/no_root_squash/root_squash/' /etc/exports && exec /usr/bin/nfsd.sh" >/dev/null
wait_for_log "${server_container_name}" "Startup successful."
server_address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${server_container_name}")"
docker volume create \
--label "${harness_label}" \
--driver local \
--opt type=nfs \
--opt "o=addr=${server_address},rw,nfsvers=4" \
--opt device=:/ \
"${client_volume_name}" >/dev/null
if docker run --rm --label "${harness_label}" --entrypoint /bin/sh \
--volume "${client_volume_name}:/export" "${databasus_image}" -c \
'chown 1:1 /export' >/dev/null 2>&1; then
echo "root-squashed NFS accepted a client-root ownership change" >&2
return 1
fi
start_databasus "${client_container_name}" \
--volume "${client_volume_name}:/databasus-data" \
--volume "${pgdata_volume_name}:/databasus-data/pgdata" \
--tmpfs /databasus-data/temp:rw,nosuid,size=64m,mode=0700,uid=999,gid=999
assert_running_container "${client_container_name}"
docker rm -fv "${client_container_name}" >/dev/null
start_databasus "${client_container_name}" \
--volume "${client_volume_name}:/databasus-data" \
--volume "${pgdata_volume_name}:/databasus-data/pgdata" \
--tmpfs /databasus-data/temp:rw,nosuid,size=64m,mode=0700,uid=999,gid=999
assert_running_container "${client_container_name}"
}
get_application_database_dsn() {
local container_name="$1"
docker exec --user databasus "${container_name}" /bin/sh -eu -c '
database_dsn="$(cat /dev/shm/databasus-database-dsn 2>/dev/null || true)"
if [ -z "${database_dsn}" ]; then
database_dsn="$(tr "\0" "\n" < /proc/1/environ | sed -n "s/^DATABASE_DSN=//p")"
fi
if [ -z "${database_dsn}" ]; then
set -a
. /.env
set +a
database_dsn="${DATABASE_DSN}"
fi
printf "%s" "${database_dsn}"
'
}
write_upgrade_markers() {
local container_name="$1"
local release_tag="$2"
local database_dsn
database_dsn="$(get_application_database_dsn "${container_name}")"
docker exec -i "${container_name}" /usr/lib/postgresql/17/bin/psql \
"${database_dsn}" -v ON_ERROR_STOP=1 -v release_tag="${release_tag}" <<'SQL'
CREATE TABLE IF NOT EXISTS filesystem_upgrade_markers (
release_tag text PRIMARY KEY
);
INSERT INTO filesystem_upgrade_markers (release_tag)
VALUES (:'release_tag')
ON CONFLICT (release_tag) DO NOTHING;
SQL
docker exec "${container_name}" /bin/sh -eu -c '
printf "%s" "$1" > /databasus-data/backups/filesystem-upgrade-marker
' sh "${release_tag}"
docker exec --user databasus "${container_name}" /bin/sh -eu -c '
printf "\nfilesystem-upgrade-log-marker:%s\n" "$1" >> /databasus-data/databasus.log
' sh "${release_tag}"
docker exec --user databasus "${container_name}" /bin/sh -eu -c '
queue_directory="/databasus-data/backups/wal-queue/filesystem-upgrade/$1"
mkdir -p "${queue_directory}/pending-upload"
printf "filesystem-upgrade-wal-marker:%s" "$1" > "${queue_directory}/pending-upload/segment"
chmod 0700 \
/databasus-data/backups/wal-queue \
/databasus-data/backups/wal-queue/filesystem-upgrade \
"${queue_directory}" \
"${queue_directory}/pending-upload"
chmod 0600 "${queue_directory}/pending-upload/segment"
' sh "${release_tag}"
}
prepare_upgrade_secret() {
local container_name="$1"
local release_tag="$2"
local database_dsn
local secret_value="filesystem-upgrade-secret-${release_tag}"
if docker exec "${container_name}" test -f /databasus-data/secret.key; then
docker exec "${container_name}" sha256sum /databasus-data/secret.key | cut -d' ' -f1
return
fi
database_dsn="$(get_application_database_dsn "${container_name}")"
docker exec "${container_name}" /usr/lib/postgresql/17/bin/psql \
"${database_dsn}" -v ON_ERROR_STOP=1 \
-c "INSERT INTO secret_keys (secret) VALUES ('${secret_value}')" >/dev/null
printf "%s" "${secret_value}" | sha256sum | cut -d' ' -f1
}
assert_upgrade_markers() {
local container_name="$1"
local release_tag="$2"
local secret_hash="$3"
local system_identifier="$4"
local log_size_before_update="$5"
local database_dsn
local log_size_after_update
local runtime_uid
database_dsn="$(get_application_database_dsn "${container_name}")"
[[ "$(docker exec "${container_name}" /usr/lib/postgresql/17/bin/psql \
"${database_dsn}" -tA -v ON_ERROR_STOP=1 \
-c "SELECT release_tag FROM filesystem_upgrade_markers WHERE release_tag='${release_tag}'")" == "${release_tag}" ]]
[[ "$(docker exec "${container_name}" cat /databasus-data/backups/filesystem-upgrade-marker)" == "${release_tag}" ]]
[[ "$(docker exec "${container_name}" sha256sum /databasus-data/secret.key | cut -d' ' -f1)" == "${secret_hash}" ]]
[[ "$(docker exec "${container_name}" /usr/lib/postgresql/17/bin/pg_controldata \
/databasus-data/pgdata | sed -n 's/^Database system identifier: *//p')" == "${system_identifier}" ]]
docker exec "${container_name}" grep -Fq \
"filesystem-upgrade-log-marker:${release_tag}" /databasus-data/databasus.log
runtime_uid="$(docker exec "${container_name}" id -u databasus)"
[[ "$(docker exec "${container_name}" stat -c %u /databasus-data/databasus.log)" == "${runtime_uid}" ]]
log_size_after_update="$(docker exec "${container_name}" stat -c %s /databasus-data/databasus.log)"
((log_size_after_update > log_size_before_update))
docker exec --user databasus "${container_name}" /bin/sh -eu -c '
queue_directory="/databasus-data/backups/wal-queue/filesystem-upgrade/$1"
test "$(cat "${queue_directory}/pending-upload/segment")" = "filesystem-upgrade-wal-marker:$1"
printf resumed > "${queue_directory}/resume-check"
rm "${queue_directory}/resume-check"
' sh "${release_tag}"
[[ "$(docker exec "${container_name}" find \
/databasus-data/backups/wal-queue -xdev ! -user "${runtime_uid}" -print -quit)" == "" ]]
}
run_upgrade() {
local release_tag="$1"
local release_image="databasus/databasus:${release_tag}"
local old_container_name="databasus-storage-upgrade-old-${release_tag}-${run_id}"
local new_container_name="databasus-storage-upgrade-new-${release_tag}-${run_id}"
local volume_name="databasus-storage-upgrade-${release_tag}-${run_id}"
local secret_hash
local system_identifier
local log_size_before_update
local had_socket_directory_before_update=false
docker image inspect "${release_image}" >/dev/null 2>&1 || docker pull "${release_image}" >/dev/null
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
docker run --detach \
--name "${old_container_name}" \
--label "${harness_label}" \
--health-interval 1s \
--health-start-period 1s \
--volume "${volume_name}:/databasus-data" \
"${release_image}" >/dev/null
wait_for_health "${old_container_name}"
write_upgrade_markers "${old_container_name}" "${release_tag}"
secret_hash="$(prepare_upgrade_secret "${old_container_name}" "${release_tag}")"
log_size_before_update="$(docker exec "${old_container_name}" stat -c %s /databasus-data/databasus.log)"
system_identifier="$(docker exec "${old_container_name}" /usr/lib/postgresql/17/bin/pg_controldata \
/databasus-data/pgdata | sed -n 's/^Database system identifier: *//p')"
if docker exec "${old_container_name}" test -d /databasus-data/pgsocket; then
had_socket_directory_before_update=true
fi
docker stop "${old_container_name}" >/dev/null
start_databasus "${new_container_name}" --volume "${volume_name}:/databasus-data"
assert_upgrade_markers \
"${new_container_name}" \
"${release_tag}" \
"${secret_hash}" \
"${system_identifier}" \
"${log_size_before_update}"
case "${release_tag}" in
v3.56.0 | v3.60.0) [[ "${had_socket_directory_before_update}" == true ]] ;;
esac
if [[ "${had_socket_directory_before_update}" == true ]]; then
docker exec "${new_container_name}" test -d /databasus-data/pgsocket
fi
assert_running_container "${new_container_name}"
}
run_upgrades() {
local release_tag
for release_tag in v3.54.0 v3.55.0 v3.55.1 v3.56.0 v3.60.0; do
run_upgrade "${release_tag}"
done
}
edit_stopped_volume() {
local volume_name="$1"
local edit_script="$2"
docker run --rm --label "${harness_label}" --entrypoint /bin/sh \
--volume "${volume_name}:/databasus-data" "${databasus_image}" -eu -c "${edit_script}"
}
create_initialized_volume() {
local volume_name="$1"
local container_name="${volume_name}-initializer"
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
start_databasus "${container_name}" --volume "${volume_name}:/databasus-data"
docker rm -fv "${container_name}" >/dev/null
}
assert_log_contains() {
local container_name="$1"
local required_log_line="$2"
if ! docker logs "${container_name}" 2>&1 | grep -F "${required_log_line}" >/dev/null; then
echo "The container log lacks: ${required_log_line}" >&2
return 1
fi
}
assert_log_lacks() {
local container_name="$1"
local forbidden_log_line="$2"
if docker logs "${container_name}" 2>&1 | grep -F "${forbidden_log_line}" >/dev/null; then
echo "The container log contains: ${forbidden_log_line}" >&2
return 1
fi
}
run_postgresql_recovery() {
local volume_name="databasus-storage-recovery-${run_id}"
local listen_volume_name="databasus-storage-recovery-listen-${run_id}"
local read_only_volume_name="databasus-storage-recovery-read-only-${run_id}"
local killed_container_name="databasus-storage-recovery-killed-${run_id}"
local replayed_container_name="databasus-storage-recovery-replayed-${run_id}"
local reset_container_name="databasus-storage-recovery-reset-${run_id}"
local unrecoverable_container_name="databasus-storage-recovery-unrecoverable-${run_id}"
local unbindable_container_name="databasus-storage-recovery-unbindable-${run_id}"
local read_only_container_name="databasus-storage-recovery-read-only-${run_id}"
local previous_password
local read_only_logs
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
start_databasus "${killed_container_name}" --volume "${volume_name}:/databasus-data"
previous_password="$(get_generated_database_password "${killed_container_name}")"
docker exec --user databasus "${killed_container_name}" /bin/sh -eu -c '
/usr/lib/postgresql/17/bin/psql "$(cat /dev/shm/databasus-database-dsn)" -v ON_ERROR_STOP=1 \
-c "CREATE TABLE filesystem_recovery_markers (marker text PRIMARY KEY)" \
-c "INSERT INTO filesystem_recovery_markers VALUES ('\''survives-kill'\'')"
' >/dev/null
docker kill --signal KILL "${killed_container_name}" >/dev/null
docker rm -fv "${killed_container_name}" >/dev/null
start_databasus "${replayed_container_name}" --volume "${volume_name}:/databasus-data"
[[ "$(docker exec --user databasus "${replayed_container_name}" /bin/sh -eu -c '
/usr/lib/postgresql/17/bin/psql "$(cat /dev/shm/databasus-database-dsn)" -tA \
-c "SELECT marker FROM filesystem_recovery_markers"
')" == "survives-kill" ]]
assert_log_lacks "${replayed_container_name}" "pg_resetwal"
if can_log_in_over_loopback "${replayed_container_name}" "${previous_password}"; then
echo "PostgreSQL accepted the password of the previous run" >&2
return 1
fi
can_log_in_over_loopback "${replayed_container_name}" \
"$(get_generated_database_password "${replayed_container_name}")"
docker stop "${replayed_container_name}" >/dev/null
docker rm -fv "${replayed_container_name}" >/dev/null
edit_stopped_volume "${volume_name}" '
find /databasus-data/pgdata/pg_wal -maxdepth 1 -type f -name "000000*" -delete
'
start_databasus "${reset_container_name}" --volume "${volume_name}:/databasus-data"
assert_log_contains "${reset_container_name}" "PostgreSQL recovered successfully after WAL reset!"
docker stop "${reset_container_name}" >/dev/null
docker rm -fv "${reset_container_name}" >/dev/null
edit_stopped_volume "${volume_name}" 'printf "16\n" > /databasus-data/pgdata/PG_VERSION'
start_failing_databasus "${unrecoverable_container_name}" --volume "${volume_name}:/databasus-data"
assert_log_contains "${unrecoverable_container_name}" \
"ERROR: PostgreSQL could not open the database even after a WAL reset attempt."
assert_log_lacks "${unrecoverable_container_name}" "Starting Databasus application..."
create_initialized_volume "${listen_volume_name}"
edit_stopped_volume "${listen_volume_name}" '
echo "listen_addresses = '\''192.0.2.1'\''" >> /databasus-data/pgdata/postgresql.conf
'
start_failing_databasus "${unbindable_container_name}" --volume "${listen_volume_name}:/databasus-data"
assert_log_contains "${unbindable_container_name}" "Setting the internal PostgreSQL password..."
assert_log_lacks "${unbindable_container_name}" "ERROR: PostgreSQL rejected the internal password change."
assert_log_contains "${unbindable_container_name}" "Starting PostgreSQL..."
assert_log_contains "${unbindable_container_name}" "ERROR: PostgreSQL did not start."
assert_log_lacks "${unbindable_container_name}" "pg_resetwal"
assert_log_lacks "${unbindable_container_name}" "Starting Databasus application..."
create_initialized_volume "${read_only_volume_name}"
edit_stopped_volume "${read_only_volume_name}" '
echo "default_transaction_read_only = on" >> /databasus-data/pgdata/postgresql.conf
'
start_failing_databasus "${read_only_container_name}" --volume "${read_only_volume_name}:/databasus-data"
assert_log_contains "${read_only_container_name}" "ERROR: PostgreSQL rejected the internal password change."
assert_log_lacks "${read_only_container_name}" "Starting PostgreSQL..."
assert_log_lacks "${read_only_container_name}" "Starting Databasus application..."
read_only_logs="$(docker logs "${read_only_container_name}" 2>&1)"
if grep -Fq "ALTER ROLE postgres WITH PASSWORD" <<<"${read_only_logs}" ||
grep -Fq "PASSWORD '" <<<"${read_only_logs}" ||
grep -Eq '[0-9a-f]{64}' <<<"${read_only_logs}"; then
echo "The container log contains the password statement" >&2
return 1
fi
}
get_cluster_state() {
local volume_name="$1"
docker run --rm --label "${harness_label}" \
--entrypoint /usr/lib/postgresql/17/bin/pg_controldata \
--volume "${volume_name}:/databasus-data" "${databasus_image}" \
/databasus-data/pgdata | sed -n 's/^Database cluster state: *//p'
}
assert_cluster_shut_down() {
local volume_name="$1"
local cluster_state
cluster_state="$(get_cluster_state "${volume_name}")"
if [[ "${cluster_state}" != "shut down" ]]; then
echo "The stopped cluster reports the state '${cluster_state}', not 'shut down'" >&2
return 1
fi
}
assert_exit_code() {
local container_name="$1"
local required_exit_code="$2"
local exit_code
exit_code="$(get_exit_code "${container_name}")"
if [[ "${exit_code}" != "${required_exit_code}" ]]; then
docker logs "${container_name}" >&2
echo "The container exited with status ${exit_code}, not ${required_exit_code}" >&2
return 1
fi
}
start_databasus_on_new_volume() {
local container_name="$1"
docker volume create --label "${harness_label}" "${container_name}" >/dev/null
start_databasus "${container_name}" --volume "${container_name}:/databasus-data"
}
check_docker_stop_shuts_down_cleanly() {
local container_name="databasus-storage-clean-stop-${run_id}"
local stop_started_at_us
local stop_duration_ms
start_databasus_on_new_volume "${container_name}"
# The uutils date on the Namespace runners printed nanoseconds for %3N, so the clock comes
# from bash instead.
stop_started_at_us="${EPOCHREALTIME/./}"
docker stop "${container_name}" >/dev/null
stop_duration_ms="$(((${EPOCHREALTIME/./} - stop_started_at_us) / 1000))"
if ((stop_duration_ms >= 10000)); then
echo "docker stop took ${stop_duration_ms} ms, so the stop timeout ended the container" >&2
return 1
fi
assert_exit_code "${container_name}" 0
assert_cluster_shut_down "${container_name}"
assert_application_exit_precedes_postgresql_shutdown "${container_name}"
}
get_first_log_line_number() {
local container_name="$1"
local searched_log_line="$2"
docker logs "${container_name}" 2>&1 | grep -nF "${searched_log_line}" | head -1 | cut -d: -f1
}
assert_application_exit_precedes_postgresql_shutdown() {
local container_name="$1"
local application_exit_line_number
local postgresql_shutdown_line_number
application_exit_line_number="$(get_first_log_line_number "${container_name}" \
"Databasus application exited with status 0")"
postgresql_shutdown_line_number="$(get_first_log_line_number "${container_name}" \
"received fast shutdown request")"
if [[ -z "${application_exit_line_number}" || -z "${postgresql_shutdown_line_number}" ]] ||
((application_exit_line_number > postgresql_shutdown_line_number)); then
docker logs "${container_name}" >&2
echo "The application exit is not logged before the PostgreSQL shutdown request" >&2
return 1
fi
}
check_interrupt_shuts_down_cleanly() {
local container_name="databasus-storage-clean-interrupt-${run_id}"
start_databasus_on_new_volume "${container_name}"
docker kill --signal INT "${container_name}" >/dev/null
wait_for_stop "${container_name}"
assert_cluster_shut_down "${container_name}"
}
check_application_exit_shuts_down_cleanly() {
local container_name="databasus-storage-clean-application-exit-${run_id}"
start_databasus_on_new_volume "${container_name}"
send_signal_in_container "${container_name}" TERM "$(docker exec "${container_name}" pidof main)"
wait_for_stop "${container_name}"
assert_exit_code "${container_name}" 0
assert_cluster_shut_down "${container_name}"
}
check_postgresql_exit_stops_container() {
local container_name="databasus-storage-clean-postgresql-exit-${run_id}"
start_databasus_on_new_volume "${container_name}"
send_signal_in_container "${container_name}" KILL \
"$(docker exec "${container_name}" head -1 /databasus-data/pgdata/postmaster.pid)"
wait_for_exit "${container_name}"
assert_log_contains "${container_name}" "ERROR: PostgreSQL exited with status"
assert_log_lacks "${container_name}" "Databasus application exited with status"
}
run_clean_shutdown() {
check_docker_stop_shuts_down_cleanly
check_interrupt_shuts_down_cleanly
check_application_exit_shuts_down_cleanly
check_postgresql_exit_stops_container
}
# The loop polls without a delay, because the open step of a clean cluster lasts about 0.1 seconds.
# A single-user step started by initdb works on template1, so matching the final postgres argument
# skips it.
# shellcheck disable=SC2016 # the container's shell expands the script
readonly postgresql_freeze_script='
target_kind="$1"
while :; do
for process_directory in /proc/[0-9]*; do
mapfile -d "" -t process_arguments < "${process_directory}/cmdline" 2>/dev/null || continue
[[ "${process_arguments[0]:-}" == /usr/lib/postgresql/17/bin/postgres ]] || continue
process_id="${process_directory#/proc/}"
case "${target_kind}" in
single-user)
[[ "${process_arguments[1]:-}" == --single &&
"${process_arguments[-1]}" == postgres ]] || continue
;;
regular-server)
[[ "${process_arguments[1]:-}" != --single ]] || continue
IFS= read -r postmaster_pid < /databasus-data/pgdata/postmaster.pid 2>/dev/null || continue
[[ "${postmaster_pid}" == "${process_id}" ]] || continue
;;
esac
kill -STOP "${process_id}"
printf "%s" "${process_id}" > /tmp/frozen-postgresql-pid
exit 0
done
done
'
start_databasus_with_frozen_postgresql() {
local container_name="$1"
local target_kind="$2"
local frozen_pid
docker volume create --label "${harness_label}" "${container_name}" >/dev/null
docker run --detach \
--name "${container_name}" \
--label "${harness_label}" \
--volume "${container_name}:/databasus-data" \
"${databasus_image}" >/dev/null
docker exec --detach "${container_name}" \
/bin/bash -c "${postgresql_freeze_script}" freeze-postgresql "${target_kind}"
for ((freeze_attempt = 1; freeze_attempt <= 300; freeze_attempt++)); do
if frozen_pid="$(docker exec "${container_name}" cat /tmp/frozen-postgresql-pid 2>/dev/null)"; then
echo "${frozen_pid}"
return
fi
sleep 0.1
done
docker logs "${container_name}" >&2
echo "No ${target_kind} PostgreSQL process appeared to freeze" >&2
return 1
}
send_signal_in_container() {
local container_name="$1"
local signal_name="$2"
local process_id="$3"
docker exec "${container_name}" /bin/bash -c 'kill -"$1" "$2"' send-signal "${signal_name}" "${process_id}"
}
is_process_stopped() {
local container_name="$1"
local process_id="$2"
[[ "$(docker exec "${container_name}" awk '/^State:/ { print $2 }' "/proc/${process_id}/status")" == "T" ]]
}
wait_for_log_within_two_seconds() {
local container_name="$1"
local required_log_line="$2"
for ((log_attempt = 1; log_attempt <= 20; log_attempt++)); do
if docker logs "${container_name}" 2>&1 | grep -F "${required_log_line}" >/dev/null; then
return
fi
sleep 0.1
done
docker logs "${container_name}" >&2
echo "The container did not log within two seconds: ${required_log_line}" >&2
return 1
}
wait_for_kill_event() {
local container_name="$1"
local container_created_at
container_created_at="$(docker inspect --format '{{.Created}}' "${container_name}")"
for ((event_attempt = 1; event_attempt <= 50; event_attempt++)); do
if docker events \
--since "${container_created_at}" \
--until "$(date +%s.%N)" \
--filter "container=${container_name}" \
--filter event=kill \
--format '{{.Action}}' | grep -q .; then
return
fi
sleep 0.1
done
echo "Docker did not report sending the stop signal" >&2
return 1
}
# Status 143 comes from the startup stop handler, while a kill by the stop timeout gives 137.
assert_startup_ended_without_application() {
local container_name="$1"
assert_exit_code "${container_name}" 143
assert_log_lacks "${container_name}" "Starting Databasus application..."
}
check_stop_during_single_user_step() {
local container_name="databasus-storage-startup-stop-single-user-${run_id}"
local frozen_pid
local stop_pid
frozen_pid="$(start_databasus_with_frozen_postgresql "${container_name}" single-user)"
docker stop "${container_name}" >/dev/null &
stop_pid=$!
wait_for_log_within_two_seconds "${container_name}" "Stop requested during startup"
if ! is_process_stopped "${container_name}" "${frozen_pid}"; then
echo "The single-user step ended before the container reported the stop" >&2
return 1
fi
send_signal_in_container "${container_name}" CONT "${frozen_pid}"
wait "${stop_pid}"
assert_startup_ended_without_application "${container_name}"
}
# The script may be waiting in pg_isready or psql on the frozen server, and Bash runs a trap only
# after that call returns, so this part cannot see the stop before the server resumes.
check_stop_while_regular_server_starts() {
local container_name="databasus-storage-startup-stop-regular-server-${run_id}"
local frozen_pid
local stop_pid
frozen_pid="$(start_databasus_with_frozen_postgresql "${container_name}" regular-server)"
docker stop "${container_name}" >/dev/null &
stop_pid=$!
wait_for_kill_event "${container_name}"
send_signal_in_container "${container_name}" CONT "${frozen_pid}"
wait "${stop_pid}"
assert_startup_ended_without_application "${container_name}"
assert_cluster_shut_down "${container_name}"
}
run_startup_stop() {
check_stop_during_single_user_step
check_stop_while_regular_server_starts
}
run_legacy_directory_guard() {
local container_name="databasus-storage-legacy-directory-${run_id}"
local volume_name="databasus-storage-legacy-directory-${run_id}"
docker volume create --label "${harness_label}" "${volume_name}" >/dev/null
docker run --rm --label "${harness_label}" --entrypoint /bin/sh \
--volume "${volume_name}:/legacy" "${databasus_image}" \
-c 'printf legacy > /legacy/marker'
start_failing_databasus "${container_name}" --volume "${volume_name}:/postgresus-data"
docker logs "${container_name}" 2>&1 | grep -F "ERROR: Legacy volume detected!" >/dev/null
}
run_case() {
local case_name="$1"
echo "Running Docker filesystem case: ${case_name}"
case "${case_name}" in
build-only) docker image inspect "${databasus_image}" >/dev/null ;;
default-ids) run_default_ids ;;
custom-ids) run_custom_ids ;;
maximum-ids) run_maximum_ids ;;
automatic-ids) run_automatic_ids ;;
empty-pgdata-ids) run_empty_pgdata_ids ;;
partial-ids) run_partial_ids ;;
invalid-ids) run_invalid_ids ;;
bind-root) run_bind_root ;;
named-volume) run_named_volume ;;
split-mounts) run_split_mounts ;;
no-unix-sockets) run_no_unix_sockets ;;
metadata-changes-denied) run_metadata_changes_denied ;;
read-only) run_read_only ;;
wrong-owner) run_wrong_owner ;;
unwritable-data-root) run_unwritable_data_root ;;
network-prerequisites) run_network_prerequisites ;;
cifs) run_cifs ;;
nfs-root-squash) run_nfs_root_squash ;;
upgrades) run_upgrades ;;
postgresql-recovery) run_postgresql_recovery ;;
clean-shutdown) run_clean_shutdown ;;
startup-stop) run_startup_stop ;;
legacy-directory-guard) run_legacy_directory_guard ;;
*)
echo "unknown Docker filesystem case: ${case_name}" >&2
return 2
;;
esac
}
# CI runs each group as a separate job, and CASE=all runs every group in this order.
readonly case_group_names=(permissions network upgrades lifecycle)
declare -rA case_groups=(
[permissions]="build-only default-ids custom-ids maximum-ids automatic-ids empty-pgdata-ids partial-ids
invalid-ids bind-root named-volume split-mounts no-unix-sockets metadata-changes-denied read-only
wrong-owner unwritable-data-root"
[network]="cifs nfs-root-squash"
[upgrades]="upgrades"
[lifecycle]="postgresql-recovery clean-shutdown startup-stop legacy-directory-guard"
)
run_cases() {
local case_name
for case_name in "$@"; do
run_case "${case_name}"
cleanup
case_root="$(mktemp -d /tmp/databasus-storage-test.XXXXXXXX)"
done
}
if [[ -n "${CASE_GROUP:-}" ]]; then
if [[ -z "${case_groups[${CASE_GROUP}]+set}" ]]; then
echo "unknown Docker filesystem case group: ${CASE_GROUP} (known: ${case_group_names[*]})" >&2
exit 2
fi
# shellcheck disable=SC2086 # the group is a whitespace-separated list of case names
run_cases ${case_groups[${CASE_GROUP}]}
elif [[ "${selected_case}" == all ]]; then
for group_name in "${case_group_names[@]}"; do
# shellcheck disable=SC2086 # the group is a whitespace-separated list of case names
run_cases ${case_groups[${group_name}]}
done
else
run_case "${selected_case}"
fi