PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (docker): Run the embedded PostgreSQL without a Unix socket and set its password in single-user mode, so a data folder that cannot hold sockets, such as a macOS Docker Desktop bind mount, no longer stops the container and a healthy cluster no longer gets its WAL reset
Rostislav Dugin013c89fd56d2
diff --git a/docker/start.sh b/docker/start.shindex c41a0840..5c804dbb 100755--- a/docker/start.sh+++ b/docker/start.sh@@ -186,7 +186,6 @@ create_required_data_directories() {for required_directory in \/databasus-data \/databasus-data/pgdata \- /databasus-data/pgsocket \/databasus-data/temp \/databasus-data/backups; doif mkdir -p "${required_directory}" 2>/dev/null ||@@ -204,12 +203,10 @@ normalize_data_permissions() {chmod 0770 /databasus-data 2>/dev/null || truechown "${runtime_uid}:${runtime_gid}" \/databasus-data/pgdata \- /databasus-data/pgsocket \/databasus-data/temp \/databasus-data/backups 2>/dev/null || truechmod 0700 \/databasus-data/pgdata \- /databasus-data/pgsocket \/databasus-data/temp 2>/dev/null || truechmod 0770 /databasus-data/backups 2>/dev/null || true}@@ -329,8 +326,7 @@ verify_service_directories() {for required_write_directory in \/databasus-data \- /databasus-data/pgdata \- /databasus-data/pgsocket; do+ /databasus-data/pgdata; doif verify_directory_file_lifecycle "${required_write_directory}" service-check; thencontinuefi@@ -371,57 +367,51 @@ initialize_postgresql_cluster() {} >> /databasus-data/pgdata/postgresql.conf}-write_bootstrap_postgresql_authentication() {- gosu databasus tee /databasus-data/pgdata/pg_ident.conf.new >/dev/null <<'PG_IDENT'-databasus_bootstrap databasus postgres-PG_IDENT- gosu databasus chmod 0600 /databasus-data/pgdata/pg_ident.conf.new- gosu databasus mv \- /databasus-data/pgdata/pg_ident.conf.new \- /databasus-data/pgdata/pg_ident.conf+install_postgresql_configuration_file() {+ local configuration_file="/databasus-data/pgdata/$1"++ gosu databasus tee "${configuration_file}.new" >/dev/null+ gosu databasus chmod 0600 "${configuration_file}.new"+ gosu databasus mv "${configuration_file}.new" "${configuration_file}"+}- gosu databasus tee /databasus-data/pgdata/pg_hba.conf.new >/dev/null <<'PG_HBA'-local all postgres peer map=databasus_bootstrap-local all all reject-local replication all reject+write_postgresql_authentication() {+ install_postgresql_configuration_file pg_hba.conf <<'PG_HBA'host all all 127.0.0.1/32 scram-sha-256host all all ::1/128 scram-sha-256host replication all 127.0.0.1/32 rejecthost replication all ::1/128 rejectPG_HBA- gosu databasus chmod 0600 /databasus-data/pgdata/pg_hba.conf.new- gosu databasus mv \- /databasus-data/pgdata/pg_hba.conf.new \- /databasus-data/pgdata/pg_hba.conf+ install_postgresql_configuration_file pg_ident.conf </dev/null}-start_postgresql() {- echo "Starting PostgreSQL..."+# Single-user mode listens on nothing, so the cluster can be administered before any client can+# connect. The database name must be the last argument.+run_postgresql_single_user() {gosu databasus "${postgres_binary_directory}/postgres" \+ --single \-D /databasus-data/pgdata \- -p 5437 \- -k /databasus-data/pgsocket \- -c password_encryption=scram-sha-256 &- postgres_pid=$!+ "$@" \+ postgres+}- echo "Waiting for PostgreSQL to be ready..."- local postgres_readiness_attempt- for ((postgres_readiness_attempt = 1; postgres_readiness_attempt <= 30; postgres_readiness_attempt++)); do- if gosu databasus "${postgres_binary_directory}/pg_isready" \- -p 5437 -h /databasus-data/pgsocket >/dev/null 2>&1; then- echo "PostgreSQL is ready!"- return- fi- sleep 1- done+# Opening the cluster replays the write-ahead log, so a failure here is the only sign of a log that+# cannot be recovered without a reset.+open_postgresql_cluster() {+ local single_user_output+ if single_user_output="$(run_postgresql_single_user </dev/null 2>&1)"; then+ return+ fi++ printf '%s\n' "${single_user_output}"return 1}report_postgresql_recovery_failure() {echo ""echo "=========================================="- echo "ERROR: PostgreSQL failed to start even after WAL reset."+ echo "ERROR: PostgreSQL could not open the database even after a WAL reset attempt."echo "The database may be severely corrupted."echo ""echo "Options:"@@ -431,37 +421,26 @@ report_postgresql_recovery_failure() {return 1}-start_postgresql_with_recovery() {- if start_postgresql; then+open_postgresql_cluster_with_recovery() {+ echo "Opening PostgreSQL database..."+ if open_postgresql_cluster; thenreturnfiecho ""echo "=========================================="- echo "PostgreSQL failed to start. Attempting WAL reset recovery..."+ echo "PostgreSQL could not open the database. Attempting WAL reset recovery..."echo "=========================================="echo ""- kill -9 "${postgres_pid}" 2>/dev/null || true- sleep 2+ # A single-user backend that panics leaves its lock file behind, and pg_resetwal refuses to run+ # while one exists. No PostgreSQL process runs at this point, so the file is stale.+ gosu databasus rm -f /databasus-data/pgdata/postmaster.pidecho "Running pg_resetwal to reset WAL..."if ! gosu databasus "${postgres_binary_directory}/pg_resetwal" \- -f /databasus-data/pgdata; then- echo ""- echo "=========================================="- echo "ERROR: pg_resetwal failed."- echo "The database may be severely corrupted."- echo ""- echo "Options:"- echo " 1. Delete the volume and start fresh (data loss)"- echo " 2. Manually inspect /databasus-data/pgdata for issues"- echo "=========================================="- return 1- fi-- echo "WAL reset successful. Restarting PostgreSQL..."- if ! start_postgresql; then+ -f /databasus-data/pgdata ||+ ! open_postgresql_cluster; thenreport_postgresql_recovery_failurereturn 1fi@@ -469,58 +448,72 @@ start_postgresql_with_recovery() {echo "PostgreSQL recovered successfully after WAL reset!"}-configure_postgresql_database() {+# printf is a shell builtin, so the password reaches PostgreSQL through a pipe and never through+# the arguments of a process. log_min_error_statement=panic keeps a failing statement, and the+# password inside it, out of the output.+rotate_internal_postgres_password() {+ local single_user_output+internal_postgres_password="$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')"- echo "Setting up database and user..."- gosu databasus "${postgres_binary_directory}/psql" \+ echo "Setting the internal PostgreSQL password..."+ if single_user_output="$(printf "ALTER ROLE postgres WITH PASSWORD '%s'\n" "${internal_postgres_password}" |+ run_postgresql_single_user \+ -c exit_on_error=on \+ -c log_min_error_statement=panic \+ -c password_encryption=scram-sha-256 2>&1)"; then+ return+ fi++ printf '%s\n' "${single_user_output}" >&2+ echo "ERROR: PostgreSQL rejected the internal password change." >&2+ return 1+}++start_postgresql() {+ echo "Starting PostgreSQL..."+ gosu databasus "${postgres_binary_directory}/postgres" \+ -D /databasus-data/pgdata \+ -p 5437 \+ -c unix_socket_directories='' \+ -c max_wal_senders=0 \+ -c password_encryption=scram-sha-256 &+ postgres_pid=$!++ echo "Waiting for PostgreSQL to be ready..."+ local postgres_readiness_attempt+ for ((postgres_readiness_attempt = 1; postgres_readiness_attempt <= 30; postgres_readiness_attempt++)); do+ if gosu databasus "${postgres_binary_directory}/pg_isready" \+ -h localhost -p 5437 >/dev/null 2>&1; then+ echo "PostgreSQL is ready!"+ return+ fi+ if ! kill -0 "${postgres_pid}" 2>/dev/null; then+ break+ fi+ sleep 1+ done++ echo "ERROR: PostgreSQL did not start." >&2+ return 1+}++create_metadata_database() {+ echo "Setting up database..."+ PGPASSWORD="${internal_postgres_password}" gosu databasus "${postgres_binary_directory}/psql" \-v ON_ERROR_STOP=1 \- -v internal_postgres_password="${internal_postgres_password}" \+ -h localhost \-p 5437 \- -h /databasus-data/pgsocket \-U postgres \-d postgres <<'SQL'--ALTER USER postgres WITH PASSWORD :'internal_postgres_password';SELECT 'CREATE DATABASE databasus OWNER postgres'WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'databasus')\gexec-\qSQL}-activate_runtime_postgresql_authentication() {- gosu databasus tee /databasus-data/pgdata/pg_hba.conf.new >/dev/null <<'PG_HBA'-local all all reject-local replication all reject-host all all 127.0.0.1/32 scram-sha-256-host all all ::1/128 scram-sha-256-host replication all 127.0.0.1/32 reject-host replication all ::1/128 reject-PG_HBA- gosu databasus chmod 0600 /databasus-data/pgdata/pg_hba.conf.new- gosu databasus mv \- /databasus-data/pgdata/pg_hba.conf.new \- /databasus-data/pgdata/pg_hba.conf- gosu databasus /bin/sh -c ': > /databasus-data/pgdata/pg_ident.conf'- gosu databasus chmod 0600 /databasus-data/pgdata/pg_ident.conf- gosu databasus "${postgres_binary_directory}/pg_ctl" \- -D /databasus-data/pgdata reload-}-verify_postgresql_runtime_authentication() {- if gosu databasus "${postgres_binary_directory}/psql" \- -p 5437 \- -h /databasus-data/pgsocket \- -U postgres \- -d postgres \- -c 'SELECT 1' >/dev/null 2>&1; then- echo "ERROR: Embedded PostgreSQL still accepts Unix-socket login after bootstrap." >&2- return 1- fi-- if ! gosu databasus env PGPASSWORD="${internal_postgres_password}" \- "${postgres_binary_directory}/psql" \+ if ! PGPASSWORD="${internal_postgres_password}" gosu databasus "${postgres_binary_directory}/psql" \-h localhost \-p 5437 \-U postgres \@@ -545,10 +538,11 @@ configure_application_database_dsn() {bootstrap_postgresql() {initialize_postgresql_cluster- write_bootstrap_postgresql_authentication- start_postgresql_with_recovery- configure_postgresql_database- activate_runtime_postgresql_authentication+ write_postgresql_authentication+ open_postgresql_cluster_with_recovery+ rotate_internal_postgres_password+ start_postgresql+ create_metadata_databaseverify_postgresql_runtime_authenticationconfigure_application_database_dsn}@@ -558,8 +552,8 @@ reject_legacy_wal_configuration() {local has_legacy_wal_databaseecho "Checking for legacy WAL backup configuration..."- has_backup_type_column="$(gosu databasus env PGPASSWORD="${internal_postgres_password}" \- "${postgres_binary_directory}/psql" \+ has_backup_type_column="$(PGPASSWORD="${internal_postgres_password}" \+ gosu databasus "${postgres_binary_directory}/psql" \-h localhost -p 5437 -U postgres -d databasus -tA \-c "SELECT 1 FROM information_schema.columns WHERE table_name='postgresql_databases' AND column_name='backup_type' LIMIT 1" \2>/dev/null || true)"@@ -569,8 +563,8 @@ reject_legacy_wal_configuration() {returnfi- has_legacy_wal_database="$(gosu databasus env PGPASSWORD="${internal_postgres_password}" \- "${postgres_binary_directory}/psql" \+ has_legacy_wal_database="$(PGPASSWORD="${internal_postgres_password}" \+ gosu databasus "${postgres_binary_directory}/psql" \-h localhost -p 5437 -U postgres -d databasus -tA \-c "SELECT 1 FROM postgresql_databases WHERE backup_type='WAL_V1' LIMIT 1" \2>/dev/null || true)"diff --git a/e2e/docker-storage/no-unix-sockets.json b/e2e/docker-storage/no-unix-sockets.jsonnew file mode 100644index 00000000..32be5d23--- /dev/null+++ b/e2e/docker-storage/no-unix-sockets.json@@ -0,0 +1,17 @@+{+ "defaultAction": "SCMP_ACT_ALLOW",+ "syscalls": [+ {+ "names": ["socket"],+ "action": "SCMP_ACT_ERRNO",+ "errnoRet": 97,+ "args": [+ {+ "index": 0,+ "value": 1,+ "op": "SCMP_CMP_EQ"+ }+ ]+ }+ ]+}diff --git a/e2e/docker-storage/run.sh b/e2e/docker-storage/run.shindex 9f174933..76ae94e6 100755--- a/e2e/docker-storage/run.sh+++ b/e2e/docker-storage/run.sh@@ -9,6 +9,8 @@ readonly selected_case="${CASE:-all}"readonly databasus_image="${DATABASUS_IMAGE:?DATABASUS_IMAGE must be set}"readonly samba_fixture_image="dockurr/samba@sha256:dbd8ebd6392cc1e8e98312d96d163f0039b644b84aee741d89efdd0abd3d32bf"readonly nfs_fixture_image="itsthenetwork/nfs-server-alpine@sha256:7fa99ae65c23c5af87dd4300e543a86b119ed15ba61422444207efc7abd0ba20"+suite_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+readonly suite_directorycase_root=""@@ -216,20 +218,34 @@ assert_single_runtime_identity() {[[ "$(docker exec "${container_name}" stat -c %g "/proc/${service_pid}")" == "${runtime_gid}" ]]}+get_generated_database_password() {+ local container_name="$1"++ docker exec "${container_name}" \+ sed -n 's/.* password=\([^ ]*\) .*/\1/p' /dev/shm/databasus-database-dsn+}++can_log_in_over_loopback() {+ local container_name="$1"+ local database_password="$2"++ docker exec -i --user databasus "${container_name}" /bin/sh -c '+ IFS= read -r PGPASSWORD+ export PGPASSWORD+ /usr/lib/postgresql/17/bin/psql -h localhost -p 5437 -U postgres -d postgres -c "SELECT 1"+ ' <<<"${database_password}" >/dev/null 2>&1+}+assert_postgresql_runtime_authentication() {local container_name="$1"+ local database_password+ local replication_probe_output- grep -Fxq "local all all reject" < <(- docker exec "${container_name}" cat /databasus-data/pgdata/pg_hba.conf- )[[ "$(docker exec "${container_name}" stat -c %a /databasus-data/pgdata/pg_hba.conf)" == "600" ]][[ "$(docker exec "${container_name}" stat -c %a /databasus-data/pgdata/pg_ident.conf)" == "600" ]]- if docker exec --user databasus "${container_name}" \- /usr/lib/postgresql/17/bin/psql \- -h /databasus-data/pgsocket -p 5437 -U postgres -d postgres \- -c 'SELECT 1' >/dev/null 2>&1; then- echo "PostgreSQL accepted a runtime Unix-socket login" >&2+ if docker exec "${container_name}" grep -Fq ".s.PGSQL" /proc/net/unix; then+ echo "PostgreSQL created a Unix socket" >&2return 1fi@@ -240,6 +256,31 @@ assert_postgresql_runtime_authentication() {echo "PostgreSQL accepted a loopback login without its generated password" >&2return 1fi++ database_password="$(get_generated_database_password "${container_name}")"+ [[ -n "${database_password}" ]]+ if docker logs "${container_name}" 2>&1 | grep -F "${database_password}" >/dev/null; then+ echo "The container log contains the generated PostgreSQL password" >&2+ return 1+ fi+ if docker exec "${container_name}" /bin/sh -c 'cat /proc/[0-9]*/cmdline 2>/dev/null' |+ tr '\0' ' ' | grep -F "${database_password}" >/dev/null; then+ echo "A process in the container carries the generated PostgreSQL password in its arguments" >&2+ return 1+ fi++ replication_probe_output="$(docker exec -i --user databasus "${container_name}" /bin/sh -c '+ IFS= read -r PGPASSWORD+ export PGPASSWORD+ /usr/lib/postgresql/17/bin/psql \+ "host=localhost port=5437 user=postgres dbname=postgres replication=database" \+ -c IDENTIFY_SYSTEM+ ' <<<"${database_password}" 2>&1 || true)"+ if ! grep -Fq "number of requested standby connections exceeds" \+ <<<"${replication_probe_output}"; then+ echo "PostgreSQL did not reject a replication connection: ${replication_probe_output}" >&2+ return 1+ fi}assert_running_container() {@@ -260,11 +301,11 @@ prepare_bind_tree() {mkdir -p "${data_root}"docker run --rm --label "${harness_label}" --entrypoint /bin/sh \--volume "${data_root}:/data" "${databasus_image}" -eu -c '- mkdir -p /data/pgdata /data/pgsocket /data/temp /data/backups- chown "$1:$2" /data /data/pgdata /data/pgsocket /data/temp+ mkdir -p /data/pgdata /data/temp /data/backups+ chown "$1:$2" /data /data/pgdata /data/tempchown "$3:$4" /data/backupschmod 0770 /data /data/backups- chmod 0700 /data/pgdata /data/pgsocket /data/temp+ chmod 0700 /data/pgdata /data/temp' sh "${data_uid}" "${data_gid}" "${backup_uid}" "${backup_gid}"}@@ -419,6 +460,17 @@ run_split_mounts() {assert_running_container "${container_name}"}+run_no_unix_sockets() {+ local container_name="databasus-storage-no-unix-sockets-${run_id}"+ local volume_name="databasus-storage-no-unix-sockets-${run_id}"++ docker volume create --label "${harness_label}" "${volume_name}" >/dev/null+ start_databasus "${container_name}" \+ --security-opt "seccomp=${suite_directory}/no-unix-sockets.json" \+ --volume "${volume_name}:/databasus-data"+ assert_running_container "${container_name}"+}+run_metadata_changes_denied() {local container_name="databasus-storage-metadata-denied-${run_id}"local data_root="${case_root}/metadata-denied"@@ -568,9 +620,9 @@ run_nfs_root_squash() {docker volume create --label "${harness_label}" "${pgdata_volume_name}" >/dev/nulldocker run --rm --label "${harness_label}" --entrypoint /bin/sh \--volume "${server_volume_name}:/nfsshare" "${databasus_image}" -c \- 'mkdir -p /nfsshare/pgdata /nfsshare/pgsocket /nfsshare/temp &&+ 'mkdir -p /nfsshare/pgdata /nfsshare/temp &&chown -R 999:999 /nfsshare &&- chmod 0755 /nfsshare /nfsshare/pgdata /nfsshare/pgsocket /nfsshare/temp'+ chmod 0755 /nfsshare /nfsshare/pgdata /nfsshare/temp'docker run --detach \--name "${server_container_name}" \--label "${harness_label}" \@@ -602,14 +654,12 @@ run_nfs_root_squash() {start_databasus "${client_container_name}" \--volume "${client_volume_name}:/databasus-data" \--volume "${pgdata_volume_name}:/databasus-data/pgdata" \- --tmpfs /databasus-data/pgsocket:rw,nosuid,size=16m,mode=0700,uid=999,gid=999 \--tmpfs /databasus-data/temp:rw,nosuid,size=64m,mode=0700,uid=999,gid=999assert_running_container "${client_container_name}"docker rm -fv "${client_container_name}" >/dev/nullstart_databasus "${client_container_name}" \--volume "${client_volume_name}:/databasus-data" \--volume "${pgdata_volume_name}:/databasus-data/pgdata" \- --tmpfs /databasus-data/pgsocket:rw,nosuid,size=16m,mode=0700,uid=999,gid=999 \--tmpfs /databasus-data/temp:rw,nosuid,size=64m,mode=0700,uid=999,gid=999assert_running_container "${client_container_name}"}@@ -725,6 +775,7 @@ run_upgrade() {local secret_hashlocal system_identifierlocal log_size_before_update+ local had_socket_directory_before_update=falsedocker image inspect "${release_image}" >/dev/null 2>&1 || docker pull "${release_image}" >/dev/nulldocker volume create --label "${harness_label}" "${volume_name}" >/dev/null@@ -741,6 +792,9 @@ run_upgrade() {log_size_before_update="$(docker exec "${old_container_name}" stat -c %s /databasus-data/databasus.log)"system_identifier="$(docker exec "${old_container_name}" /usr/lib/postgresql/17/bin/pg_controldata \/databasus-data/pgdata | sed -n 's/^Database system identifier: *//p')"+ if docker exec "${old_container_name}" test -d /databasus-data/pgsocket; then+ had_socket_directory_before_update=true+ fidocker stop "${old_container_name}" >/dev/nullstart_databasus "${new_container_name}" --volume "${volume_name}:/databasus-data"@@ -750,17 +804,142 @@ run_upgrade() {"${secret_hash}" \"${system_identifier}" \"${log_size_before_update}"+ case "${release_tag}" in+ v3.56.0 | v3.60.0) [[ "${had_socket_directory_before_update}" == true ]] ;;+ esac+ if [[ "${had_socket_directory_before_update}" == true ]]; then+ docker exec "${new_container_name}" test -d /databasus-data/pgsocket+ fiassert_running_container "${new_container_name}"}run_upgrades() {local release_tag- for release_tag in v3.54.0 v3.55.0 v3.55.1 v3.56.0; do+ for release_tag in v3.54.0 v3.55.0 v3.55.1 v3.56.0 v3.60.0; dorun_upgrade "${release_tag}"done}+edit_stopped_volume() {+ local volume_name="$1"+ local edit_script="$2"++ docker run --rm --label "${harness_label}" --entrypoint /bin/sh \+ --volume "${volume_name}:/databasus-data" "${databasus_image}" -eu -c "${edit_script}"+}++create_initialized_volume() {+ local volume_name="$1"+ local container_name="${volume_name}-initializer"++ docker volume create --label "${harness_label}" "${volume_name}" >/dev/null+ start_databasus "${container_name}" --volume "${volume_name}:/databasus-data"+ docker rm -fv "${container_name}" >/dev/null+}++assert_log_contains() {+ local container_name="$1"+ local required_log_line="$2"++ if ! docker logs "${container_name}" 2>&1 | grep -F "${required_log_line}" >/dev/null; then+ echo "The container log lacks: ${required_log_line}" >&2+ return 1+ fi+}++assert_log_lacks() {+ local container_name="$1"+ local forbidden_log_line="$2"++ if docker logs "${container_name}" 2>&1 | grep -F "${forbidden_log_line}" >/dev/null; then+ echo "The container log contains: ${forbidden_log_line}" >&2+ return 1+ fi+}++run_postgresql_recovery() {+ local volume_name="databasus-storage-recovery-${run_id}"+ local listen_volume_name="databasus-storage-recovery-listen-${run_id}"+ local read_only_volume_name="databasus-storage-recovery-read-only-${run_id}"+ local killed_container_name="databasus-storage-recovery-killed-${run_id}"+ local replayed_container_name="databasus-storage-recovery-replayed-${run_id}"+ local reset_container_name="databasus-storage-recovery-reset-${run_id}"+ local unrecoverable_container_name="databasus-storage-recovery-unrecoverable-${run_id}"+ local unbindable_container_name="databasus-storage-recovery-unbindable-${run_id}"+ local read_only_container_name="databasus-storage-recovery-read-only-${run_id}"+ local previous_password+ local read_only_logs++ docker volume create --label "${harness_label}" "${volume_name}" >/dev/null+ start_databasus "${killed_container_name}" --volume "${volume_name}:/databasus-data"+ previous_password="$(get_generated_database_password "${killed_container_name}")"+ docker exec --user databasus "${killed_container_name}" /bin/sh -eu -c '+ /usr/lib/postgresql/17/bin/psql "$(cat /dev/shm/databasus-database-dsn)" -v ON_ERROR_STOP=1 \+ -c "CREATE TABLE filesystem_recovery_markers (marker text PRIMARY KEY)" \+ -c "INSERT INTO filesystem_recovery_markers VALUES ('\''survives-kill'\'')"+ ' >/dev/null+ docker kill --signal KILL "${killed_container_name}" >/dev/null+ docker rm -fv "${killed_container_name}" >/dev/null++ start_databasus "${replayed_container_name}" --volume "${volume_name}:/databasus-data"+ [[ "$(docker exec --user databasus "${replayed_container_name}" /bin/sh -eu -c '+ /usr/lib/postgresql/17/bin/psql "$(cat /dev/shm/databasus-database-dsn)" -tA \+ -c "SELECT marker FROM filesystem_recovery_markers"+ ')" == "survives-kill" ]]+ assert_log_lacks "${replayed_container_name}" "pg_resetwal"+ if can_log_in_over_loopback "${replayed_container_name}" "${previous_password}"; then+ echo "PostgreSQL accepted the password of the previous run" >&2+ return 1+ fi+ can_log_in_over_loopback "${replayed_container_name}" \+ "$(get_generated_database_password "${replayed_container_name}")"+ docker stop "${replayed_container_name}" >/dev/null+ docker rm -fv "${replayed_container_name}" >/dev/null++ edit_stopped_volume "${volume_name}" '+ find /databasus-data/pgdata/pg_wal -maxdepth 1 -type f -name "000000*" -delete+ '+ start_databasus "${reset_container_name}" --volume "${volume_name}:/databasus-data"+ assert_log_contains "${reset_container_name}" "PostgreSQL recovered successfully after WAL reset!"+ docker stop "${reset_container_name}" >/dev/null+ docker rm -fv "${reset_container_name}" >/dev/null++ edit_stopped_volume "${volume_name}" 'printf "16\n" > /databasus-data/pgdata/PG_VERSION'+ start_failing_databasus "${unrecoverable_container_name}" --volume "${volume_name}:/databasus-data"+ assert_log_contains "${unrecoverable_container_name}" \+ "ERROR: PostgreSQL could not open the database even after a WAL reset attempt."+ assert_log_lacks "${unrecoverable_container_name}" "Starting Databasus application..."++ create_initialized_volume "${listen_volume_name}"+ edit_stopped_volume "${listen_volume_name}" '+ echo "listen_addresses = '\''192.0.2.1'\''" >> /databasus-data/pgdata/postgresql.conf+ '+ start_failing_databasus "${unbindable_container_name}" --volume "${listen_volume_name}:/databasus-data"+ assert_log_contains "${unbindable_container_name}" "Setting the internal PostgreSQL password..."+ assert_log_lacks "${unbindable_container_name}" "ERROR: PostgreSQL rejected the internal password change."+ assert_log_contains "${unbindable_container_name}" "Starting PostgreSQL..."+ assert_log_contains "${unbindable_container_name}" "ERROR: PostgreSQL did not start."+ assert_log_lacks "${unbindable_container_name}" "pg_resetwal"+ assert_log_lacks "${unbindable_container_name}" "Starting Databasus application..."++ create_initialized_volume "${read_only_volume_name}"+ edit_stopped_volume "${read_only_volume_name}" '+ echo "default_transaction_read_only = on" >> /databasus-data/pgdata/postgresql.conf+ '+ start_failing_databasus "${read_only_container_name}" --volume "${read_only_volume_name}:/databasus-data"+ assert_log_contains "${read_only_container_name}" "ERROR: PostgreSQL rejected the internal password change."+ assert_log_lacks "${read_only_container_name}" "Starting PostgreSQL..."+ assert_log_lacks "${read_only_container_name}" "Starting Databasus application..."+ read_only_logs="$(docker logs "${read_only_container_name}" 2>&1)"+ if grep -Fq "ALTER ROLE postgres WITH PASSWORD" <<<"${read_only_logs}" ||+ grep -Fq "PASSWORD '" <<<"${read_only_logs}" ||+ grep -Eq '[0-9a-f]{64}' <<<"${read_only_logs}"; then+ echo "The container log contains the password statement" >&2+ return 1+ fi+}+run_legacy_directory_guard() {local container_name="databasus-storage-legacy-directory-${run_id}"local volume_name="databasus-storage-legacy-directory-${run_id}"@@ -789,6 +968,7 @@ run_case() {bind-root) run_bind_root ;;named-volume) run_named_volume ;;split-mounts) run_split_mounts ;;+ no-unix-sockets) run_no_unix_sockets ;;metadata-changes-denied) run_metadata_changes_denied ;;read-only) run_read_only ;;wrong-owner) run_wrong_owner ;;@@ -797,6 +977,7 @@ run_case() {cifs) run_cifs ;;nfs-root-squash) run_nfs_root_squash ;;upgrades) run_upgrades ;;+ postgresql-recovery) run_postgresql_recovery ;;legacy-directory-guard) run_legacy_directory_guard ;;*)echo "unknown Docker filesystem case: ${case_name}" >&2@@ -808,9 +989,9 @@ run_case() {if [[ "${selected_case}" == all ]]; thenfor case_name in \build-only default-ids custom-ids maximum-ids automatic-ids empty-pgdata-ids partial-ids invalid-ids \- bind-root named-volume split-mounts metadata-changes-denied read-only wrong-owner \+ bind-root named-volume split-mounts no-unix-sockets metadata-changes-denied read-only wrong-owner \unwritable-data-root \- cifs nfs-root-squash upgrades legacy-directory-guard; do+ cifs nfs-root-squash upgrades postgresql-recovery legacy-directory-guard; dorun_case "${case_name}"cleanupcase_root="$(mktemp -d /tmp/databasus-storage-test.XXXXXXXX)"diff --git a/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/.openspec.yaml b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/.openspec.yamlnew file mode 100644index 00000000..dc36fb1b--- /dev/null+++ b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/.openspec.yaml@@ -0,0 +1,2 @@+schema: spec-driven+created: 2026-10-21diff --git a/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/design.md b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/design.mdnew file mode 100644index 00000000..509b336c--- /dev/null+++ b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/design.md@@ -0,0 +1,116 @@+## Context++See `proposal.md` for the motivation. The embedded PostgreSQL server is started with `-k /databasus-data/pgsocket` and waits for readiness through that socket (`docker/start.sh:398-419`). Bootstrap writes a peer mapping from the `databasus` account to role `postgres` (`write_bootstrap_postgresql_authentication`, `docker/start.sh:374-396`). It then sets the generated password through the socket (`configure_postgresql_database`, `docker/start.sh:472-490`), replaces the authentication rules with `local all all reject` and reloads the server (`activate_runtime_postgresql_authentication`, `docker/start.sh:492-509`). Finally it checks that the socket refuses a login (`verify_postgresql_runtime_authentication`, `docker/start.sh:511-532`). The socket therefore carries one administrative statement per start. Every other connection, including the application's, uses loopback TCP with SCRAM.++The socket location has moved twice. An earlier image used the default `/var/run/postgresql`. ZFS-backed overlay storage on TrueNAS Scale ignores ownership changes on directories from image layers, so PostgreSQL could not create its lock file there. Commit `27d891fb` moved the socket to `/tmp`. Commit `61fe8dd8` then moved it under the data root, because `/tmp` is reachable by every process in the container. The archived `protect-internal-postgresql-access` design (decision 6) states the placement without comparing locations. It rejected disabling sockets because "Startup would need another bootstrap credential path". Single-user mode was not considered.++Since `simplify-docker-runtime-user`, PostgreSQL and the application share one operating-system account. The `0700` mode on the socket directory therefore separates PostgreSQL from other UIDs only, not from the application. The authentication rules already provide the effective barrier.++Current startup also exposes the generated password in process arguments. `psql -v internal_postgres_password=...` (`docker/start.sh:474-476`) and the `gosu databasus env PGPASSWORD=...` calls (`docker/start.sh:522`, `docker/start.sh:561`, `docker/start.sh:572`) put it into the argument list of a running process.++The approach answers to the root `AGENTS.md`: no compatibility shims, refactoring of the touched startup code within the same change, and the test rules of the testing skill. ADR-0003 keeps PostgreSQL embedded in the image, and this change stays inside that decision. No module `AGENTS.md` governs `docker/` or `e2e/`.++## Goals / Non-Goals++**Goals:**++- The embedded server creates no Unix socket at any point, so no mount property can block startup.+- No startup step admits a login without the generated password.+- The generated password appears in no process argument list and in no container log line.+- The filesystem suite reproduces the macOS failure on a Linux CI runner.++**Non-Goals:**++- Hardening of the password in process environments. The application account can already read the published connection string, so hiding the environment from the same account protects nothing.+- Changing how the backend classifies socket hosts.++## Decisions++### 1. Administer the cluster in single-user mode before the server starts++Startup runs `postgres --single` under the `databasus` account against the data directory. It feeds `ALTER ROLE postgres WITH PASSWORD '<generated>'` through standard input. Single-user mode opens the cluster files directly and listens on nothing. Nothing else can connect while the password changes, and the server never runs with an old or unknown password.++A spike against PostgreSQL 17.11 confirmed the behavior this decision relies on:++- The statement succeeds without any listener. No `.s.PGSQL` file appears in `/tmp` or `/var/run/postgresql`.+- After an unclean shutdown caused by `kill -9`, single-user mode replays the write-ahead log and commits the statement. All 100000 rows written before the kill survived.+- Single-user mode refuses to run while a postmaster holds `postmaster.pid`.+- With a missing checkpoint record it aborts with `PANIC: could not locate a valid checkpoint record`.++Alternatives rejected:++- Move the socket to `/dev/shm`, which is tmpfs, writable under a read-only root filesystem and already holds the published connection string (`docker/start.sh:13`). This fixes macOS with fewer changes. It keeps the peer-mapping window and a socket file whose permissions protect nothing against the shared account.+- Move the socket to `/run/databasus`. A directory baked into the image repeats the TrueNAS ownership failure. A directory created at startup needs `chown`, which the hardened Helm values lack because they drop every capability except `SETUID` and `SETGID` (`deploy/helm/README.md:398-409`). A read-only root filesystem also makes `/run` unwritable without an extra volume.+- Use an abstract Unix socket. It has no file-system permissions. With `--network host` it lives in the host's network namespace, where any host process can reach it and two containers collide on the name.+- Trust loopback TCP during bootstrap. With `--network host`, loopback belongs to the host, so any host process could log in without a password during that window.+- Persist a password with `initdb --pwfile`. It covers only new clusters, and the archived design deliberately avoided storing a secret on the volume.+- Keep the socket under the data root and document a tmpfs mount for macOS. Every macOS user would hit the failure before finding the workaround.++### 2. Split single-user work into an open step and a password step++The first single-user invocation receives empty input. It only opens the cluster, which replays the write-ahead log when needed, and then exits with a shutdown checkpoint. A failure of this step is the only trigger for `pg_resetwal`, after which the open step runs once more. Before the reset, startup removes `postmaster.pid`. A single-user backend that panics, as it does on a missing checkpoint record, leaves its lock file behind, and `pg_resetwal` refuses to run while that file exists. No PostgreSQL process runs at that point, so the file is stale. When `pg_resetwal` or that second attempt fails, startup prints the existing recovery-failure message from `report_postgresql_recovery_failure` and exits. The second invocation sets the password with `-c exit_on_error=on`, so a failing statement ends the backend with exit code 1 instead of 0. In the spike, exit code 0 after a failed statement was the default behavior. A failure of the password step stops the container without a reset, because the cluster has already opened successfully.++Today any start failure triggers `pg_resetwal`, including a socket failure on a healthy cluster. That is what happened in the reported macOS log. After this change a reset happens only when the cluster cannot be opened at all. A regular server start that fails after both steps succeeded stops the container with an error.++Alternatives rejected:++- One invocation that opens the cluster and sets the password. Its exit code cannot tell a broken log from a failed statement. Telling them apart would require parsing PostgreSQL output.+- Keep the reset around the regular server start. Once single-user mode has opened the cluster, the log is consistent, and a reset at that point can only discard data.++### 3. Keep the password out of arguments and the log++The shell writes the statement with the `printf` builtin into a pipe to the single-user backend. A builtin spawns no process, so the password never enters an argument list. The password step runs with `-c log_min_error_statement=panic`. With the default setting `error`, the spike showed the failing `ALTER ROLE` statement, password included, in the `STATEMENT:` log line. With `panic` the statement was absent. On failure, the password step prints the captured output to standard error. PostgreSQL's error text can name the command, as in `cannot execute ALTER ROLE in a read-only transaction`, but it contains neither the statement nor the password. `-c password_encryption=scram-sha-256` is passed to the password step as well, so the stored verifier matches the server's SCRAM rules. The spike confirmed a `SCRAM-SHA-256` verifier.++`psql` calls that need the password get `PGPASSWORD` as a shell assignment prefix in front of `gosu`, for example `PGPASSWORD="${internal_postgres_password}" gosu databasus psql ...`. `gosu` preserves the environment, so the password reaches `psql` without appearing in the arguments of `gosu` or `env`.++Alternatives rejected:++- `psql -v` variables, as today. They are argument values.+- A temporary `PGPASSFILE`. It adds a secret file and a cleanup path for no gain over the environment of a short-lived process.++### 4. Write the authentication rules once and start without sockets++Before any PostgreSQL process starts, startup writes the final `pg_hba.conf`: SCRAM on `127.0.0.1/32` and `::1/128`, and reject for replication. It also writes an empty `pg_ident.conf`. Both files get mode `0600` as today. The regular server starts with `-c unix_socket_directories=''` beside the existing `-p 5437` and `-c password_encryption=scram-sha-256`. The server also starts with `-c max_wal_senders=0`. The `host replication` reject rules match only physical replication. A logical replication connection, requested with `replication=database`, is matched by database name, so it falls under the SCRAM rule, and the superuser `postgres` may replicate. With no WAL senders, PostgreSQL refuses every replication connection after authentication. The command line applies these settings to new and existing clusters alike, because `postgresql.conf` is only appended on `initdb` (`docker/start.sh:366-371`). Readiness polls `pg_isready -h localhost -p 5437`, which needs no authentication.++The `local` rules disappear from `pg_hba.conf`, since no socket exists to match them. The bootstrap peer map, the rule rewrite, `pg_ctl reload` and the check that a socket login fails are deleted. The check that the generated password works over loopback stays.++The metadata database is created after the server starts, through `psql -h localhost` with the generated password, using the existing `\gexec` statement.++Alternatives rejected:++- Keep `local all all reject` as defense in depth. With sockets disabled the rule can never match, and it would suggest a socket exists.+- Revoke replication with `ALTER ROLE postgres NOREPLICATION`. A superuser bypasses that attribute.+- Write `unix_socket_directories` into `postgresql.conf`. Existing clusters would need an edit of a file that startup otherwise only appends on initialization.++### 5. Stop preparing the socket directory++`create_required_data_directories`, `normalize_data_permissions` and `verify_service_directories` drop `/databasus-data/pgsocket` (`docker/start.sh:182-213`, `docker/start.sh:327-339`). A directory left by an earlier image is neither touched nor deleted.++Alternative rejected: delete the leftover directory. Removal is a mutation of user data with no benefit, and it would fail on mounts that deny it.++### 6. Reproduce the macOS failure with seccomp++A new filesystem case starts the candidate with a seccomp profile. The profile allows every system call except `socket` with address family `AF_UNIX`, which returns `EAFNOSUPPORT`. In the spike, a locally built image with the current startup script failed under it with the same `could not create any Unix-domain sockets` error as on macOS. PostgreSQL 17 started under the profile without sockets and answered over TCP.++A shared assertion checks that `/proc/net/unix` inside the container lists no `.s.PGSQL` entry. It runs in every case through `assert_running_container`, so every layout proves that no socket exists without hard-coding a directory. The NFS case loses its tmpfs mounts on `/databasus-data/pgsocket`. `prepare_bind_tree` and the NFS share setup stop creating the directory. The runtime authentication assertion drops its socket-login probe and keeps the loopback probe without a password.++The upgrade cases add `v3.60.0`, the latest release that bootstraps through the socket.++Alternatives rejected:++- Run Docker Desktop on macOS in CI. Runners would need nested virtualization and a licensed Docker Desktop, and the job would test one platform instead of the general property.+- Mount a FUSE or vfat file system as the data root. Such a mount needs host privileges and brings unrelated failures, such as missing ownership, into the case.++## Risks / Trade-offs++- [A bundled component other than PostgreSQL needs `AF_UNIX`, so the seccomp case fails for an unrelated reason] -> The current image dies at the PostgreSQL start under the profile, so such a component can only surface in task 2.5, when the rewritten bootstrap lets the application run. If it does, the case narrows the profile or is redesigned there, before the filesystem suite changes in group 3.+- [Single-user mode adds a shutdown checkpoint per invocation, two per start] -> The spike measured under 0.1 s per checkpoint on a fresh cluster. After an unclean shutdown, recovery runs once in the open step, as it ran in the regular start before.+- [`pg_resetwal` still runs for an open failure that has nothing to do with the log, such as a permission error] -> Storage validation runs earlier and covers the data directory. Narrowing the trigger further is out of scope.+- [Operators lose socket access for manual administration] -> They use `psql` with the connection string the container publishes, or single-user mode in a stopped container.+- [The single-user backend prints a banner and `backend>` prompts to the container log] -> Its output is discarded unless the step fails. On failure, startup prints the captured output. The statement is absent from it because of `log_min_error_statement=panic`. The error text may still name the command, such as `ALTER ROLE`.++## Migration Plan++1. Build the image and run the new no-socket case, the bind-root case and the upgrade cases locally.+2. Publish the image. Existing volumes start without manual steps. The leftover socket directory stays empty.+3. Rolling back to `v3.60.0` needs no volume change. That image recreates its socket directory and rewrites `pg_hba.conf` with its own peer rule before it starts the server (`v3.60.0:docker/start.sh:546-554`). Rollback to older images keeps the guidance of the archived `protect-internal-postgresql-access` design.diff --git a/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/proposal.md b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/proposal.mdnew file mode 100644index 00000000..2e1a98a5--- /dev/null+++ b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/proposal.md@@ -0,0 +1,50 @@+## Why++The embedded PostgreSQL server creates its Unix socket inside the mounted data directory, at `/databasus-data/pgsocket`. Docker Desktop on macOS shares a host folder through a file system that cannot set permissions on a socket file. A user who bind-mounts a host folder as `/databasus-data` therefore gets `could not create any Unix-domain sockets`, and the container never starts. The same failure applies to any data mount without Unix-socket support.++The socket serves one purpose. At startup it lets the image set a new random password on the internal `postgres` role through peer authentication. After that step, authentication rules reject every socket login. Setting the password in PostgreSQL single-user mode needs neither a socket nor a network listener, so the server can run without a socket at all.++## Governing docs++This change answers to [`AGENTS.md`](../../../AGENTS.md). It changes `docker/start.sh`, the Docker filesystem suite in `e2e/docker-storage/` and two OpenSpec capabilities. It changes no code under `backend/`, `frontend/`, `agent/` or `website/`.++## What Changes++- Startup sets the generated password on the internal `postgres` role in PostgreSQL single-user mode, before the server accepts any connection. Neither a socket nor a TCP listener exists during that step.+- The embedded PostgreSQL server runs with Unix sockets disabled. It listens only on loopback TCP, with SCRAM authentication and the generated password from its first accepted connection.+- Startup creates the metadata database over loopback TCP with the generated password.+- The embedded server starts with WAL senders disabled, so it refuses logical replication connections as well as physical ones. Authentication rules alone rejected only physical replication.+- The temporary peer mapping is deleted, together with its `pg_ident.conf` entry, the authentication rewrite and the reload that followed bootstrap. The authentication rules are written once, before the server starts.+- Startup no longer creates, prepares or verifies `/databasus-data/pgsocket`. An existing directory from an earlier image is left in place and ignored.+- The `pg_resetwal` recovery runs when the single-user step cannot open the cluster, because that step replays the write-ahead log first. A later failure of the regular server start stops the container without resetting the write-ahead log.+- The password no longer appears in the arguments of a startup process. It reaches PostgreSQL through standard input, and a failing statement is kept out of the container log.+- The Docker filesystem suite gains a case in which the container cannot create Unix sockets at all. The current image fails that case exactly as on macOS. The workaround mounts for the socket directory in the NFS case are deleted. The upgrade cases add `v3.60.0`, the latest release that bootstraps through the socket.+- **BREAKING**: an operator can no longer administer the embedded cluster through the image's peer-authenticated socket. The rollback procedure in the archived `protect-internal-postgresql-access` design relied on it. Operators use the generated connection string inside a running container instead, or single-user mode inside a stopped one.++## Capabilities++### New Capabilities++None.++### Modified Capabilities++- `internal-postgresql-protection`: the embedded server accepts no Unix-socket connections, and bootstrap administration happens with no listener at all instead of through temporary peer mapping.+- `docker-storage-compatibility`: storage validation no longer covers a PostgreSQL socket path. Startup succeeds when the data mount cannot hold Unix sockets, and the filesystem matrix covers that case. Upgrades cover `v3.60.0`. A write-ahead log reset happens only when the cluster cannot be opened.++## Impact++- `docker/start.sh`: PostgreSQL bootstrap, start, readiness check, recovery path and storage preparation.+- `e2e/docker-storage/run.sh`: runtime authentication assertions, bind-tree preparation, the NFS case and a new case without Unix-socket support.+- OpenSpec specs `internal-postgresql-protection` and `docker-storage-compatibility`.+- No backend, frontend, API, database migration, configuration variable or dependency change.++## Out of scope++- Moving temporary files, backups or PostgreSQL data off `/databasus-data`.+- Removing a leftover `/databasus-data/pgsocket` directory from existing volumes.+- Changing the Helm chart, its README or its `extraVolumes` guidance about `/var/run`.+- Narrowing when `pg_resetwal` runs beyond the move to the single-user step, such as matching specific PostgreSQL errors.+- Changing the embedded-target classifier in the backend. It keeps rejecting every Unix-socket host.+- Further compatibility work for Docker Desktop on macOS beyond the socket, such as file-system performance.+- Website or README documentation changes.diff --git a/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/specs/docker-storage-compatibility/spec.md b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/specs/docker-storage-compatibility/spec.mdnew file mode 100644index 00000000..714b21d1--- /dev/null+++ b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/specs/docker-storage-compatibility/spec.md@@ -0,0 +1,132 @@+## MODIFIED Requirements++### Requirement: Startup validates storage through required operations++Startup SHALL attempt to prepare the data root, PostgreSQL data, temporary and backup paths with the selected identity and required modes. A failed ownership or mode change SHALL NOT stop startup when the selected account can still perform every required operation.++Startup SHALL run the main Databasus binary with `--test-storage` under the selected account before starting PostgreSQL or the Databasus application. The storage probe SHALL save a unique file through the configured local storage and remove that file through the same storage path. Startup SHALL also verify the operations required for PostgreSQL data and Databasus control files. A failed check SHALL stop startup before PostgreSQL or the Databasus application runs. Startup SHALL NOT start or wait for a separate cache service.++#### Scenario: Filesystem denies metadata changes but permits operations++- **WHEN** a mounted filesystem rejects ownership or mode changes but the selected account can perform all required operations+- **THEN** the container starts successfully++#### Scenario: Selected identity creates missing service directories++- **WHEN** container root cannot create a required directory but the selected account can create it+- **THEN** startup creates the directory as the selected account and continues++#### Scenario: Backup publication is not permitted++- **WHEN** the selected account cannot publish a file from the temporary path to the backup path+- **THEN** the container exits before starting the application+- **AND** the English error names the affected path, required operation, effective user and group IDs and `https://databasus.com/advanced-config/#docker-storage-permissions`++#### Scenario: Mounted path is read-only++- **WHEN** any required mounted path is read-only+- **THEN** the container exits before starting PostgreSQL or the Databasus application+- **AND** the log provides the same identity and documentation details++#### Scenario: Data root cannot create control files++- **WHEN** temporary, backup and PostgreSQL data paths are writable but the data root cannot create a control file+- **THEN** the container exits before starting PostgreSQL or the Databasus application+- **AND** the English error identifies the data root and required operation++#### Scenario: Container starts its bundled services++- **WHEN** storage validation succeeds+- **THEN** startup proceeds directly to embedded PostgreSQL and the Databasus application without a cache-service readiness step++### Requirement: Existing Docker data remains usable after an image update++The current image SHALL start with persistent data created by versions `v3.54.0`, `v3.55.0`, `v3.55.1`, `v3.56.0` and `v3.60.0` when the mounted filesystem permits the selected account's required operations. Startup SHALL preserve the metadata database, secret key, application log, existing backup files, resumable nested WAL queues, and the existing PostgreSQL cluster. It SHALL prepare an existing WAL queue recursively and verify directory creation plus non-mutating file opens under the selected account before PostgreSQL or the Databasus application starts.++#### Scenario: Existing installation is updated++- **WHEN** an operator replaces one of the covered earlier images with the current image without adding identity settings+- **THEN** the current container becomes healthy+- **AND** the metadata database, secret key, application log, existing backup, nested WAL queue, and PostgreSQL data remain available++### Requirement: The matrix covers supported mount behavior++The suite SHALL cover an ext4 bind-mounted data root, a Docker named volume, separate temporary, backup, and PostgreSQL mounts, automatic and explicit runtime IDs, rejected backup and data-root permission layouts, CIFS numeric ownership and forced file modes, writable NFS with root squash, a container that cannot create Unix sockets, and upgrades from the covered earlier images. It SHALL verify one non-root numeric identity for all bundled services, runtime PostgreSQL authentication, the absence of a PostgreSQL Unix socket, and restrictive modes for generated credential files.++#### Scenario: Local Docker storage layouts++- **WHEN** the candidate uses an ext4 bind mount, named volume, or separate mounts+- **THEN** the container becomes healthy and the storage command saves and deletes its probe++#### Scenario: Temporary and backup paths use different filesystems++- **WHEN** the temporary and backup paths have different device IDs+- **THEN** `LocalStorage.SaveFile` succeeds through its supported cross-filesystem path and `LocalStorage.DeleteFile` removes the probe++#### Scenario: CIFS maps numeric ownership++- **WHEN** a real CIFS fixture presents root ownership, numeric group ownership `999`, directory mode `0770`, and mount-enforced file modes+- **THEN** the container becomes healthy and completes the storage command+- **AND** startup skips the root user ID and writes through group `999`++#### Scenario: NFS data storage denies ownership changes to container root++- **WHEN** a root-squashed NFS data root has no backups directory, rejects ownership and mode changes, remains writable by the selected identity, and local mounts provide PostgreSQL and temporary storage+- **THEN** the selected identity creates the backups directory and the container becomes healthy+- **AND** the storage command succeeds without requiring ownership changes++#### Scenario: Container cannot create Unix sockets++- **WHEN** the container runs with Unix-socket creation denied for every process+- **THEN** the container becomes healthy and completes the storage command++#### Scenario: Mounted path denies required access++- **WHEN** a mounted path does not permit an operation required by the selected identity+- **THEN** startup fails with the documented actionable error++## ADDED Requirements++### Requirement: Startup does not depend on Unix-socket support in the data mount++The container SHALL start when the mounted data root cannot hold Unix sockets, provided the selected account can perform every other required operation there. Startup SHALL NOT create, prepare or verify a PostgreSQL socket directory. A socket directory left by an earlier image SHALL be ignored.++#### Scenario: No Unix socket can be created in the container++- **WHEN** no process in the container can create a Unix socket, in the data root or anywhere else, as happens on a host folder that Docker Desktop on macOS shares as the data root+- **THEN** the embedded PostgreSQL server starts and the container becomes healthy+- **AND** the container lists no PostgreSQL Unix socket++#### Scenario: Socket directory from an earlier image++- **WHEN** the data root contains a PostgreSQL socket directory created by an earlier image+- **THEN** startup leaves the directory in place and ignores it+- **AND** the container becomes healthy++### Requirement: A write-ahead log reset is limited to a cluster that cannot be opened++Startup SHALL reset the embedded cluster's write-ahead log only when the cluster cannot be opened for startup administration, which includes replaying the existing log. After a reset, startup SHALL try to open the cluster once more, and SHALL exit with an error that reports the failed recovery when the reset or that attempt fails. When administration succeeds and the regular server then fails to start, the container SHALL exit with an error and SHALL NOT reset the write-ahead log.++#### Scenario: Interrupted shutdown++- **WHEN** the previous container stopped without a clean PostgreSQL shutdown and the write-ahead log is intact+- **THEN** startup replays the log and keeps every committed transaction+- **AND** the container becomes healthy without a write-ahead log reset++#### Scenario: Missing checkpoint record++- **WHEN** the cluster cannot be opened because its write-ahead log lacks a valid checkpoint record+- **THEN** the log reports a write-ahead log reset+- **AND** the container becomes healthy++#### Scenario: Reset does not make the cluster open++- **WHEN** the write-ahead log reset fails, or the cluster still cannot be opened after the reset+- **THEN** the container exits with an error before the Databasus application starts+- **AND** the log reports the failed recovery++#### Scenario: Server start fails after administration++- **WHEN** startup administration succeeds but the regular server cannot start+- **THEN** the container exits with an error+- **AND** the write-ahead log is not resetdiff --git a/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/specs/internal-postgresql-protection/spec.md b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/specs/internal-postgresql-protection/spec.mdnew file mode 100644index 00000000..980c030c--- /dev/null+++ b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/specs/internal-postgresql-protection/spec.md@@ -0,0 +1,66 @@+## ADDED Requirements++### Requirement: The embedded PostgreSQL instance accepts only authenticated loopback TCP++The embedded PostgreSQL server SHALL run under the same non-root operating-system account as the Databasus application. The server SHALL create no Unix socket, so no process in the container can open a socket connection to it. Loopback TCP connections SHALL require SCRAM authentication with the generated credential from the first connection the server accepts, and replication connections to the embedded cluster SHALL be rejected.++The embedded PostgreSQL port SHALL remain unpublished by the image.++#### Scenario: A process looks for a local socket++- **WHEN** any process in a running container attempts a Unix-socket connection to the embedded PostgreSQL server+- **THEN** the connection fails because the server exposes no socket++#### Scenario: Loopback TCP connection without the generated password++- **WHEN** any process connects to the embedded PostgreSQL server over loopback TCP without the current generated password+- **THEN** SCRAM authentication rejects the connection++#### Scenario: Application uses the generated password++- **WHEN** Databasus connects to the embedded PostgreSQL server over loopback with the generated password+- **THEN** PostgreSQL accepts the connection++#### Scenario: Replication connection to the embedded cluster++- **WHEN** a client requests a replication connection to the embedded PostgreSQL server+- **THEN** PostgreSQL rejects it++### Requirement: Startup administration of the embedded cluster opens no connection path++The container SHALL set the generated password on the internal role before the embedded PostgreSQL server accepts any connection. That step SHALL run while the cluster has no network listener and no Unix socket. No startup step SHALL grant a password-less or identity-mapped login to the embedded cluster.++#### Scenario: First connection after startup administration++- **WHEN** the embedded PostgreSQL server accepts its first connection after a container start+- **THEN** the internal role already carries the newly generated password+- **AND** the connection must authenticate with that password++#### Scenario: Password from the previous run after a restart++- **WHEN** the container restarts and a process logs in over loopback with the password published by the previous run+- **THEN** SCRAM authentication rejects the connection+- **AND** a loopback login with the newly published password succeeds++### Requirement: The generated password stays out of process arguments and server logs++Startup SHALL NOT pass the generated password to any process through its command-line arguments. When the statement that sets the password fails, the container SHALL exit with an error before the Databasus application starts, and the container log SHALL NOT contain that statement or the password.++#### Scenario: Process list during startup++- **WHEN** a process in the container reads the command-line arguments of the startup processes+- **THEN** none of them contains the generated password++#### Scenario: Password statement fails++- **WHEN** the statement that sets the generated password fails+- **THEN** the container exits before starting the Databasus application+- **AND** the container log contains neither the statement nor the generated password++## REMOVED Requirements++### Requirement: The embedded PostgreSQL instance stays private under the shared runtime account++**Reason**: The embedded server no longer creates a Unix socket, so the temporary peer mapping and the post-bootstrap socket rejection this requirement described no longer exist. "The embedded PostgreSQL instance accepts only authenticated loopback TCP" and "Startup administration of the embedded cluster opens no connection path" replace it.++**Migration**: No operator action is needed. Manual administration uses the connection string published inside a running container, or PostgreSQL single-user mode inside a stopped one.diff --git a/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/tasks.md b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/tasks.mdnew file mode 100644index 00000000..94871a40--- /dev/null+++ b/openspec/changes/archive/2026-10-21-remove-embedded-postgresql-socket/tasks.md@@ -0,0 +1,23 @@+## 1. Reproduce the failure++- [x] 1.1 Add `e2e/docker-storage/no-unix-sockets.json`, a seccomp profile that allows every system call except `socket` with address family `AF_UNIX` (value `1`), which returns errno `97`. Add a `no-unix-sockets` case to `e2e/docker-storage/run.sh` that starts the candidate with `--security-opt seccomp=<profile>` on a named volume and calls `assert_running_container`. Add the case to the `all` list. Verify that `docker build --tag databasus-storage-test:local .` and `CASE=no-unix-sockets DATABASUS_IMAGE=databasus-storage-test:local bash e2e/docker-storage/run.sh` fail, and that the container log contains `could not create any Unix-domain sockets`.++## 2. Bootstrap without a socket++- [x] 2.1 In `docker/start.sh`, drop `/databasus-data/pgsocket` from `create_required_data_directories`, `normalize_data_permissions` and `verify_service_directories`. Verify that `grep -n pgsocket docker/start.sh` lists only the lines that tasks 2.2 to 2.4 replace.+- [x] 2.2 Replace `write_bootstrap_postgresql_authentication` and `activate_runtime_postgresql_authentication` with one function. It writes the final `pg_hba.conf`, with SCRAM on `127.0.0.1/32` and `::1/128`, reject for replication and no `local` rules, and an empty `pg_ident.conf`, both with mode `0600`, before any PostgreSQL process starts. Delete the `pg_ctl reload` call. Verify that `grep -nE 'peer|pg_ctl|databasus_bootstrap' docker/start.sh` prints nothing.+- [x] 2.3 Add the single-user open step from design decision 2. It runs `postgres --single` with empty input under `databasus`, captures its output and prints that output only on failure. Move the `pg_resetwal` recovery from `start_postgresql_with_recovery` around this step, and remove a stale `postmaster.pid` before `pg_resetwal` runs. Verify after task 2.5 through the recovery case of task 3.4.+- [x] 2.4 Add the single-user password step from design decisions 2 and 3. It pipes `ALTER ROLE postgres WITH PASSWORD '<generated>'` from the `printf` builtin into `postgres --single` with `-c exit_on_error=on`, `-c log_min_error_statement=panic` and `-c password_encryption=scram-sha-256`, and stops startup on a non-zero exit. Verify that `grep -nE 'env PGPASSWORD|-v internal_postgres_password' docker/start.sh` prints nothing once task 2.5 has also converted the remaining `psql` calls.+- [x] 2.5 Start the regular server with `-c unix_socket_directories=''` and `-c max_wal_senders=0` and poll `pg_isready -h localhost -p 5437`. Without a reset, a failed start stops the container. Create the metadata database through `psql -h localhost` with the existing `\gexec` statement. Pass `PGPASSWORD` as a shell assignment prefix in front of `gosu` in every `psql` call, including `reject_legacy_wal_configuration`. Delete the socket-login check from `verify_postgresql_runtime_authentication` and keep the loopback check. Verify that `docker build --tag databasus-storage-test:local .` succeeds and that `CASE=no-unix-sockets DATABASUS_IMAGE=databasus-storage-test:local bash e2e/docker-storage/run.sh` passes. Then verify that `grep -n pgsocket docker/start.sh` prints nothing.++## 3. Filesystem suite++- [x] 3.1 In `assert_postgresql_runtime_authentication`, delete the socket-login probe and the `local all all reject` check. Add a check that `/proc/net/unix` inside the container lists no `.s.PGSQL` entry. Add a check that neither the `docker logs` output nor any `/proc/*/cmdline` inside the container contains the password from `/dev/shm/databasus-database-dsn`. Add a probe that runs `psql "host=localhost port=5437 user=postgres dbname=postgres replication=database"` with the generated password from `/dev/shm/databasus-database-dsn` and asserts that PostgreSQL rejects it with `number of requested standby connections exceeds`. Keep the loopback probe without a password and the `0600` mode checks. Verify that `CASE=bind-root DATABASUS_IMAGE=databasus-storage-test:local bash e2e/docker-storage/run.sh` and the `no-unix-sockets` case pass.+- [x] 3.2 Stop creating `pgsocket` in `prepare_bind_tree` and in the NFS share setup. Delete both `--tmpfs /databasus-data/pgsocket` mounts from the NFS case. Verify that `grep -n pgsocket e2e/docker-storage/run.sh` prints nothing, and that the `automatic-ids`, `split-mounts` and `nfs-root-squash` cases pass one at a time through `CASE=<case> DATABASUS_IMAGE=databasus-storage-test:local bash e2e/docker-storage/run.sh`.+- [x] 3.3 Add `v3.60.0` to `run_upgrades`. Add a check to `run_upgrade` that records whether `/databasus-data/pgsocket` exists before the update and, when it did, asserts that it still exists afterwards in the healthy container. Only `v3.56.0` and later create that directory, so for those releases the check requires it to exist before the update. Verify that `CASE=upgrades DATABASUS_IMAGE=databasus-storage-test:local bash e2e/docker-storage/run.sh` passes.+- [x] 3.4 Add a `postgresql-recovery` case with five parts, and add it to the `all` list. In the first part, read the password from `/dev/shm/databasus-database-dsn`, write a marker row, stop the container with `docker kill --signal KILL`, start it again and assert that it is healthy, the marker exists and the log has no `pg_resetwal` line. Assert also that a loopback `psql -h localhost` login with the old password is refused and one with the newly published password succeeds. In the second part, stop the container cleanly, remove the segments under `pgdata/pg_wal`, start it again and assert that the log reports a write-ahead log reset and that the container is healthy. In the third part, write `16` into `pgdata/PG_VERSION`, which makes both the open step and `pg_resetwal` fail, start the container and assert that it exits with an error, the application never starts and the log reports the failed recovery. Run the remaining parts on fresh volumes. In the fourth part, append `listen_addresses = '192.0.2.1'` to `postgresql.conf`, start the container and assert that it exits with an error, that the log reports the password step without its failure, `Starting PostgreSQL...` and `ERROR: PostgreSQL did not start.`, and that it has no `pg_resetwal` line. In the fifth part, append `default_transaction_read_only = on` to `postgresql.conf`, start the container and assert four things: it exits with a non-zero code, the log reports `ERROR: PostgreSQL rejected the internal password change.` and has no `Starting PostgreSQL...` line, the application never starts, and `docker logs` contains neither `ALTER ROLE postgres WITH PASSWORD`, nor `PASSWORD '`, nor any 64-character hexadecimal string. Verify that `CASE=postgresql-recovery DATABASUS_IMAGE=databasus-storage-test:local bash e2e/docker-storage/run.sh` passes.++## 4. Specs and review++- [x] 4.1 Verify that `openspec validate remove-embedded-postgresql-socket --strict` passes.+- [x] 4.2 Run the mandatory full implementation review from `AGENTS.md` and resolve its blocking findings. Then rerun the `no-unix-sockets`, `bind-root`, `upgrades` and `postgresql-recovery` cases through `CASE=<case> DATABASUS_IMAGE=databasus-storage-test:local bash e2e/docker-storage/run.sh` after rebuilding the image, and verify that they pass.diff --git a/openspec/specs/docker-storage-compatibility/spec.md b/openspec/specs/docker-storage-compatibility/spec.mdindex 3f15d5db..eeb5764a 100644--- a/openspec/specs/docker-storage-compatibility/spec.md+++ b/openspec/specs/docker-storage-compatibility/spec.md@@ -40,9 +40,9 @@ When an override is absent, startup SHALL select that ID independently from the### Requirement: Startup validates storage through required operations-Startup SHALL attempt to prepare the data root, PostgreSQL data, PostgreSQL socket, temporary and backup paths with the selected identity and required modes. A failed ownership or mode change SHALL NOT stop startup when the selected account can still perform every required operation.+Startup SHALL attempt to prepare the data root, PostgreSQL data, temporary and backup paths with the selected identity and required modes. A failed ownership or mode change SHALL NOT stop startup when the selected account can still perform every required operation.-Startup SHALL run the main Databasus binary with `--test-storage` under the selected account before starting PostgreSQL or the Databasus application. The storage probe SHALL save a unique file through the configured local storage and remove that file through the same storage path. Startup SHALL also verify the operations required for PostgreSQL data, its socket and Databasus control files. A failed check SHALL stop startup before PostgreSQL or the Databasus application runs. Startup SHALL NOT start or wait for a separate cache service.+Startup SHALL run the main Databasus binary with `--test-storage` under the selected account before starting PostgreSQL or the Databasus application. The storage probe SHALL save a unique file through the configured local storage and remove that file through the same storage path. Startup SHALL also verify the operations required for PostgreSQL data and Databasus control files. A failed check SHALL stop startup before PostgreSQL or the Databasus application runs. Startup SHALL NOT start or wait for a separate cache service.#### Scenario: Filesystem denies metadata changes but permits operations@@ -68,7 +68,7 @@ Startup SHALL run the main Databasus binary with `--test-storage` under the sele#### Scenario: Data root cannot create control files-- **WHEN** temporary, backup, PostgreSQL data and socket paths are writable but the data root cannot create a control file+- **WHEN** temporary, backup and PostgreSQL data paths are writable but the data root cannot create a control file- **THEN** the container exits before starting PostgreSQL or the Databasus application- **AND** the English error identifies the data root and required operation@@ -79,7 +79,7 @@ Startup SHALL run the main Databasus binary with `--test-storage` under the sele### Requirement: Existing Docker data remains usable after an image update-The current image SHALL start with persistent data created by versions `v3.54.0`, `v3.55.0`, `v3.55.1`, and `v3.56.0` when the mounted filesystem permits the selected account's required operations. Startup SHALL preserve the metadata database, secret key, application log, existing backup files, resumable nested WAL queues, and the existing PostgreSQL cluster. It SHALL prepare an existing WAL queue recursively and verify directory creation plus non-mutating file opens under the selected account before PostgreSQL or the Databasus application starts.+The current image SHALL start with persistent data created by versions `v3.54.0`, `v3.55.0`, `v3.55.1`, `v3.56.0` and `v3.60.0` when the mounted filesystem permits the selected account's required operations. Startup SHALL preserve the metadata database, secret key, application log, existing backup files, resumable nested WAL queues, and the existing PostgreSQL cluster. It SHALL prepare an existing WAL queue recursively and verify directory creation plus non-mutating file opens under the selected account before PostgreSQL or the Databasus application starts.#### Scenario: Existing installation is updated@@ -126,7 +126,7 @@ Each positive case SHALL start the real container and invoke `databasus --test-s### Requirement: The matrix covers supported mount behavior-The suite SHALL cover an ext4 bind-mounted data root, a Docker named volume, separate temporary, backup, and PostgreSQL mounts, automatic and explicit runtime IDs, rejected backup and data-root permission layouts, CIFS numeric ownership and forced file modes, writable NFS with root squash, and upgrades from the covered earlier images. It SHALL verify one non-root numeric identity for all bundled services, bootstrap and runtime PostgreSQL authentication, and restrictive modes for generated credential files.+The suite SHALL cover an ext4 bind-mounted data root, a Docker named volume, separate temporary, backup, and PostgreSQL mounts, automatic and explicit runtime IDs, rejected backup and data-root permission layouts, CIFS numeric ownership and forced file modes, writable NFS with root squash, a container that cannot create Unix sockets, and upgrades from the covered earlier images. It SHALL verify one non-root numeric identity for all bundled services, runtime PostgreSQL authentication, the absence of a PostgreSQL Unix socket, and restrictive modes for generated credential files.#### Scenario: Local Docker storage layouts@@ -150,7 +150,56 @@ The suite SHALL cover an ext4 bind-mounted data root, a Docker named volume, sep- **THEN** the selected identity creates the backups directory and the container becomes healthy- **AND** the storage command succeeds without requiring ownership changes+#### Scenario: Container cannot create Unix sockets++- **WHEN** the container runs with Unix-socket creation denied for every process+- **THEN** the container becomes healthy and completes the storage command+#### Scenario: Mounted path denies required access- **WHEN** a mounted path does not permit an operation required by the selected identity- **THEN** startup fails with the documented actionable error++### Requirement: Startup does not depend on Unix-socket support in the data mount++The container SHALL start when the mounted data root cannot hold Unix sockets, provided the selected account can perform every other required operation there. Startup SHALL NOT create, prepare or verify a PostgreSQL socket directory. A socket directory left by an earlier image SHALL be ignored.++#### Scenario: No Unix socket can be created in the container++- **WHEN** no process in the container can create a Unix socket, in the data root or anywhere else, as happens on a host folder that Docker Desktop on macOS shares as the data root+- **THEN** the embedded PostgreSQL server starts and the container becomes healthy+- **AND** the container lists no PostgreSQL Unix socket++#### Scenario: Socket directory from an earlier image++- **WHEN** the data root contains a PostgreSQL socket directory created by an earlier image+- **THEN** startup leaves the directory in place and ignores it+- **AND** the container becomes healthy++### Requirement: A write-ahead log reset is limited to a cluster that cannot be opened++Startup SHALL reset the embedded cluster's write-ahead log only when the cluster cannot be opened for startup administration, which includes replaying the existing log. After a reset, startup SHALL try to open the cluster once more, and SHALL exit with an error that reports the failed recovery when the reset or that attempt fails. When administration succeeds and the regular server then fails to start, the container SHALL exit with an error and SHALL NOT reset the write-ahead log.++#### Scenario: Interrupted shutdown++- **WHEN** the previous container stopped without a clean PostgreSQL shutdown and the write-ahead log is intact+- **THEN** startup replays the log and keeps every committed transaction+- **AND** the container becomes healthy without a write-ahead log reset++#### Scenario: Missing checkpoint record++- **WHEN** the cluster cannot be opened because its write-ahead log lacks a valid checkpoint record+- **THEN** the log reports a write-ahead log reset+- **AND** the container becomes healthy++#### Scenario: Reset does not make the cluster open++- **WHEN** the write-ahead log reset fails, or the cluster still cannot be opened after the reset+- **THEN** the container exits with an error before the Databasus application starts+- **AND** the log reports the failed recovery++#### Scenario: Server start fails after administration++- **WHEN** startup administration succeeds but the regular server cannot start+- **THEN** the container exits with an error+- **AND** the write-ahead log is not resetdiff --git a/openspec/specs/internal-postgresql-protection/spec.md b/openspec/specs/internal-postgresql-protection/spec.mdindex 4dbe7bef..0e27a79f 100644--- a/openspec/specs/internal-postgresql-protection/spec.md+++ b/openspec/specs/internal-postgresql-protection/spec.md@@ -164,21 +164,16 @@ Scheduled healthchecks SHALL test a database already loaded by the system withou- **WHEN** an HTTP caller requests a direct connection test- **THEN** the system uses the workspace-authorized path rather than the trusted healthcheck path-### Requirement: The embedded PostgreSQL instance stays private under the shared runtime account+### Requirement: The embedded PostgreSQL instance accepts only authenticated loopback TCP-The embedded PostgreSQL server SHALL run under the same non-root operating-system account as the Databasus application. During bootstrap only, peer mapping SHALL allow that account to administer the PostgreSQL role named `postgres`. After bootstrap, Unix-socket database login SHALL be rejected. Loopback TCP connections SHALL require SCRAM authentication with the generated credential and replication connections to the embedded cluster SHALL be rejected.+The embedded PostgreSQL server SHALL run under the same non-root operating-system account as the Databasus application. The server SHALL create no Unix socket, so no process in the container can open a socket connection to it. Loopback TCP connections SHALL require SCRAM authentication with the generated credential from the first connection the server accepts, and replication connections to the embedded cluster SHALL be rejected.The embedded PostgreSQL port SHALL remain unpublished by the image.-#### Scenario: Application account opens the private socket+#### Scenario: A process looks for a local socket-- **WHEN** the Databasus application account attempts a Unix-socket database login after bootstrap-- **THEN** PostgreSQL authentication rejects the connection--#### Scenario: Shared runtime account performs startup administration--- **WHEN** the shared `databasus` operating-system account connects as role `postgres` during startup-- **THEN** the temporary peer mapping permits the administrative connection+- **WHEN** any process in a running container attempts a Unix-socket connection to the embedded PostgreSQL server+- **THEN** the connection fails because the server exposes no socket#### Scenario: Loopback TCP connection without the generated password@@ -195,6 +190,37 @@ The embedded PostgreSQL port SHALL remain unpublished by the image.- **WHEN** a client requests a replication connection to the embedded PostgreSQL server- **THEN** PostgreSQL rejects it+### Requirement: Startup administration of the embedded cluster opens no connection path++The container SHALL set the generated password on the internal role before the embedded PostgreSQL server accepts any connection. That step SHALL run while the cluster has no network listener and no Unix socket. No startup step SHALL grant a password-less or identity-mapped login to the embedded cluster.++#### Scenario: First connection after startup administration++- **WHEN** the embedded PostgreSQL server accepts its first connection after a container start+- **THEN** the internal role already carries the newly generated password+- **AND** the connection must authenticate with that password++#### Scenario: Password from the previous run after a restart++- **WHEN** the container restarts and a process logs in over loopback with the password published by the previous run+- **THEN** SCRAM authentication rejects the connection+- **AND** a loopback login with the newly published password succeeds++### Requirement: The generated password stays out of process arguments and server logs++Startup SHALL NOT pass the generated password to any process through its command-line arguments. When the statement that sets the password fails, the container SHALL exit with an error before the Databasus application starts, and the container log SHALL NOT contain that statement or the password.++#### Scenario: Process list during startup++- **WHEN** a process in the container reads the command-line arguments of the startup processes+- **THEN** none of them contains the generated password++#### Scenario: Password statement fails++- **WHEN** the statement that sets the generated password fails+- **THEN** the container exits before starting the Databasus application+- **AND** the container log contains neither the statement nor the generated password+### Requirement: Internal PostgreSQL credentials are generated at startupWhen Databasus uses its embedded metadata database, the container SHALL generate a cryptographically random PostgreSQL password at startup, set it on the internal role, and supply it to the application without logging it. A container restart SHALL replace the previous password. An explicitly supplied external metadata-database connection string SHALL remain unchanged.