Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bind database bootstrap and ordinary CLI transfers to the selected endpoint (#10757)
[._.]/ Adam Eivy3ebc9277c56a
diff --git a/docs/STORAGE.md b/docs/STORAGE.mdindex 8012c633db..6f733cb335 100644--- a/docs/STORAGE.md+++ b/docs/STORAGE.md@@ -380,6 +380,20 @@ dump/import tools are used only for the selected local Docker endpoint; ahost/port override cannot silently fall back to the local container. Maintenance`--endpoint` transfers remain bound to their explicitly recorded endpoint.+**Endpoint binding (#10757).** The resolved host, port, user and database are+passed as explicit arguments, but libpq also honors inherited variables that+choose where a client really connects: `PGHOSTADDR` overrides the network+destination of `-h`, `PGSERVICE`/`PGSERVICEFILE` can supply another endpoint,+and `PGOPTIONS` alters session settings. Every host `psql`, `pg_dump` and+`pg_isready` that `scripts/db.sh` starts — readiness, `setup-native` role,+database, extension and schema statements, and ordinary `export`/`import` —+runs with those four variables removed from the child only (`pg_bound`), as does+the readiness probe and bootstrap child of `scripts/setup-db.js`. The operator's+shell is never modified. Supported TLS and authentication settings (`PGSSLMODE`,+`PGSSLROOTCERT`, `PGPASSFILE`, ...) pass through on these ordinary paths;+`--endpoint` maintenance transfers keep their stricter scrub of every `PG*`+variable. Passwords stay in the child environment or stdin, never argv.+### `MEMORY_BACKEND=file` is a development/test-only escape hatch — NOT a deployment modeThe file backend (`server/services/memory.js`, JSON under `./data/`) is **unsupported for production and for federated peers.** It exists only so the test suite (and ad-hoc local development) can boot without a database. It is **not** a fallback, a "lite" mode, or a way to run PortOS without Postgres:diff --git a/scripts/db-migrate.test.js b/scripts/db-migrate.test.jsindex 27b30afa07..701364caae 100644--- a/scripts/db-migrate.test.js+++ b/scripts/db-migrate.test.js@@ -45,7 +45,7 @@ exit 0const PSQL_STUB = `#!/bin/shecho "psql $*" >> "$STUB_LOG"-echo "endpoint-env \${PGHOSTADDR-unset}|\${PGSERVICE-unset}|\${PGSERVICEFILE-unset}|\${PGOPTIONS-unset}" >> "$STUB_LOG"+echo "endpoint-env \${PGHOSTADDR-unset}|\${PGSERVICE-unset}|\${PGSERVICEFILE-unset}|\${PGOPTIONS-unset}|tls=\${PGSSLMODE-unset}" >> "$STUB_LOG"case "$*" in *--single-transaction*) cat >> "$IMPORT_LOG"; exit "\${IMPORT_EXIT:-0}" ;; esacif [ -n "\${PORTOS_BOOTSTRAP_ROLE:-}" ]; then cat > "$ROLE_INPUT_LOG"; ficase "$*" in *count*) echo 3 ;; esac@@ -54,7 +54,7 @@ exit 0const PG_DUMP_STUB = `#!/bin/shecho "pg_dump $*" >> "$STUB_LOG"-echo "endpoint-env \${PGHOSTADDR-unset}|\${PGSERVICE-unset}|\${PGSERVICEFILE-unset}|\${PGOPTIONS-unset}" >> "$STUB_LOG"+echo "endpoint-env \${PGHOSTADDR-unset}|\${PGSERVICE-unset}|\${PGSERVICEFILE-unset}|\${PGOPTIONS-unset}|tls=\${PGSSLMODE-unset}" >> "$STUB_LOG"case "$DUMP_MODE" inpartial) printf 'DROP TABLE example_record;\\n'; exit 1 ;;empty) exit 1 ;;@@ -66,6 +66,7 @@ esac// test can stand up "another cluster on the default port" without a real server.const PG_ISREADY_STUB = `#!/bin/shecho "pg_isready $*" >> "$STUB_LOG"+echo "endpoint-env \${PGHOSTADDR-unset}|\${PGSERVICE-unset}|\${PGSERVICEFILE-unset}|\${PGOPTIONS-unset}|tls=\${PGSSLMODE-unset}" >> "$STUB_LOG"port=""; prev=""for arg in "$@"; do [ "$prev" = "-p" ] && port="$arg"; prev="$arg"; done[ -z "\${READY_PORTS:-}" ] && exit 0@@ -374,6 +375,55 @@ describe.skipIf(process.platform === 'win32')('scripts/db.sh export + migrate',expect(result.stdout).toContain('selected mode is unchanged');});+ // #10757: every host libpq client the script starts — readiness, bootstrap+ // probes and writes, schema, ordinary and explicit transfers — must run with+ // the routing variables removed, while supported TLS settings survive for the+ // ordinary paths. Pairing each client line with an environment line proves no+ // call site was missed.+ describe('inherited libpq routing variables (#10757)', () => {+ const poisoned = { PGHOSTADDR: '192.0.2.10', PGSERVICE: 'other', PGSERVICEFILE: '/example/service.conf', PGOPTIONS: '-c search_path=other', PGSSLMODE: 'require' };+ const clientLines = (log) => log.split('\n').filter(line => /^(psql|pg_dump|pg_isready) /.test(line));+ const envLines = (log) => log.split('\n').filter(line => line.startsWith('endpoint-env '));++ const expectRoutingRemoved = (log, { tls }) => {+ expect(envLines(log).length).toBe(clientLines(log).length);+ expect(envLines(log).length).toBeGreaterThan(0);+ for (const line of envLines(log)) expect(line).toBe(`endpoint-env unset|unset|unset|unset|tls=${tls}`);+ expect(log).not.toContain('192.0.2.10');+ };++ it('removes them from native readiness, role provisioning, database creation and schema application', () => {+ const result = run(['setup-native'], 'ok', { ...poisoned, PGPORT: '5433', PGHOST: 'db.example.invalid', READY_PORTS: '5433' });+ expect(result.status, result.stderr).toBe(0);+ const log = readFileSync(stubLog, 'utf8');+ expect(log).toContain('pg_isready');+ expect(psqlCalls().length).toBeGreaterThanOrEqual(6);+ expectRoutingRemoved(log, { tls: 'require' });+ expect(readFileSync(join(root, 'role.sql'), 'utf8')).toContain('CREATE ROLE :"role"');+ expect(log).not.toContain('test-only');+ });++ it.each(['docker', 'native'])('removes them from ordinary host export and import in saved %s mode', (mode) => {+ writeFileSync(envFile, `PGMODE=${mode}\nPGHOST=db.example.invalid\nPGPORT=6543\nPGPORT_DOCKER=6543\n`);+ const overrides = { ...poisoned, PGHOST: undefined };+ const exported = run(['export', 'ordinary'], 'ok', overrides);+ expect(exported.status, exported.stderr).toBe(0);+ const dump = join(dumpDir, 'portos-ordinary.sql');+ expect(run(['import', dump], 'ok', overrides).status).toBe(0);+ const log = readFileSync(stubLog, 'utf8');+ expect(log).toMatch(/pg_dump -h db\.example\.invalid -p 6543 /);+ expect(log).toMatch(/psql -h db\.example\.invalid -p 6543 .*--single-transaction/);+ expect(clientLines(log)).toHaveLength(2);+ expectRoutingRemoved(log, { tls: 'require' });+ });++ it('keeps explicit endpoint transfers on the stricter scrub that also drops TLS settings', () => {+ const endpoint = ['--endpoint', 'example.invalid', '6543', 'example', 'example_db'];+ expect(run(['export', ...endpoint, 'strict'], 'ok', poisoned).status).toBe(0);+ expectRoutingRemoved(readFileSync(stubLog, 'utf8'), { tls: 'unset' });+ });+ });+const psqlCalls = () => readFileSync(stubLog, 'utf8').split('\n').filter(line => line.startsWith('psql '));it('provisions only the selected non-default endpoint while another cluster listens on the default port', () => {diff --git a/scripts/db.sh b/scripts/db.shindex 39a75d4bda..c07b363687 100755--- a/scripts/db.sh+++ b/scripts/db.sh@@ -92,6 +92,26 @@ warn() { echo -e "${YELLOW}⚠️ $1${NC}"; }err() { echo -e "${RED}❌ $1${NC}"; }info() { echo -e "${BLUE}🗄️ $1${NC}"; }+# Inherited libpq variables that choose WHERE a client connects or how the+# session behaves, independent of the -h/-p/-U/-d arguments: PGHOSTADDR+# overrides -h's network destination, PGSERVICE/PGSERVICEFILE can supply another+# endpoint, and PGOPTIONS alters session settings (search_path). Keep this list+# in step with ROUTING_ENV in scripts/setup-db.js. TLS and authentication+# variables (PGSSLMODE, PGSSLROOTCERT, PGPASSFILE, ...) are deliberately NOT+# listed: they are supported operator configuration, not endpoint identity.+PG_ROUTING_VARS="PGHOSTADDR PGSERVICE PGSERVICEFILE PGOPTIONS"++# Run a host libpq client (psql, pg_dump, pg_isready) with those variables+# removed from ITS environment only; the operator's shell is untouched. Every+# call site passes the resolved host/port/user/database as explicit arguments.+pg_bound() (+ local name+ for name in $PG_ROUTING_VARS; do+ unset "$name"+ done+ "$@"+)+get_mode() {printf '%s\n' "$PGMODE"}@@ -166,7 +186,7 @@ require_docker_compose() {# Check if native PostgreSQL is accepting connections on the expected portnative_running() {- PGPASSWORD="$PGPASSWORD" pg_isready -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" >/dev/null 2>&1+ PGPASSWORD="$PGPASSWORD" pg_bound pg_isready -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" >/dev/null 2>&1}# Auto-detect Homebrew PostgreSQL 17 on macOS and add to PATH@@ -188,7 +208,7 @@ has_native_pg() {# Detect an already-running system PostgreSQL and its portdetect_system_pg() {# Check standard port 5432 first- if pg_isready -h localhost -p 5432 >/dev/null 2>&1; then+ if pg_bound pg_isready -h localhost -p 5432 >/dev/null 2>&1; thenecho "5432"return 0fi@@ -239,7 +259,7 @@ cmd_status() {if sys_port=$(detect_system_pg); thenlog " System PostgreSQL is running on port $sys_port"# Check if portos database exists- if PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$sys_port" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1" >/dev/null 2>&1; then+ if PGPASSWORD="$PGPASSWORD" pg_bound psql -h "$PGHOST" -p "$sys_port" -U "$PGUSER" -d "$PGDATABASE" -c "SELECT 1" >/dev/null 2>&1; thenlog " PortOS database exists"elsewarn " PortOS database/user not configured (run: scripts/db.sh setup-native)"@@ -320,7 +340,7 @@ start_native() {fi# Check if system PostgreSQL is already running and accepting connections- if PGPASSWORD="$PGPASSWORD" pg_isready -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" >/dev/null 2>&1; then+ if native_running; thenlog "Native PostgreSQL already running on port $PGPORT"returnfi@@ -331,7 +351,7 @@ start_native() {info "Starting PostgreSQL via Homebrew services..."brew services start postgresql@17 2>/dev/null || truefor i in $(seq 1 15); do- if pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; then+ if pg_bound pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; thenlog "Native PostgreSQL ready on port $PGPORT"returnfi@@ -352,7 +372,7 @@ start_native() {if [ -n "$datadir" ] && [ -d "$datadir" ]; thenpg_ctl -D "$datadir" -l "$datadir/server.log" start 2>/dev/null || truefor i in $(seq 1 15); do- if pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; then+ if pg_bound pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; thenlog "Native PostgreSQL ready on port $PGPORT"returnfi@@ -469,7 +489,7 @@ provision_role() {local sys_user="$1" statement="$2"printf '%s\n%s\n%s\n' '\getenv role PORTOS_BOOTSTRAP_ROLE' '\getenv pw PORTOS_BOOTSTRAP_PASSWORD' "$statement" |PORTOS_BOOTSTRAP_ROLE="$PGUSER" PORTOS_BOOTSTRAP_PASSWORD="$PGPASSWORD" \- psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -v ON_ERROR_STOP=1 -X -q+ pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -v ON_ERROR_STOP=1 -X -q}cmd_setup_native() {@@ -512,13 +532,13 @@ cmd_setup_native() {# (role/password changes, schema) to a different cluster than the one chosen.if [ -n "$SELECTED_PGPORT" ] || [ -n "$SELECTED_PGHOST" ]; thenPGPORT="${SELECTED_PGPORT:-5432}"- if pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; then+ if pg_bound pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; thenlog "PostgreSQL already running at $PGHOST:$PGPORT"elif [ "$(uname)" = "Darwin" ] && command -v brew >/dev/null 2>&1; theninfo "Starting PostgreSQL..."brew services start postgresql@17sleep 2- if pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; then+ if pg_bound pg_isready -h "$PGHOST" -p "$PGPORT" >/dev/null 2>&1; thenlog "PostgreSQL started at $PGHOST:$PGPORT"elseerr "PostgreSQL is not accepting connections at $PGHOST:$PGPORT. Start the selected cluster and try again."@@ -555,7 +575,7 @@ cmd_setup_native() {# Connect as the current system user (default Homebrew superuser) to create the rolelocal sys_usersys_user="$(whoami)"- if ! psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$PGUSER'" 2>/dev/null | grep -q 1; then+ if ! pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$PGUSER'" 2>/dev/null | grep -q 1; theninfo "Creating database user: $PGUSER"provision_role "$sys_user" 'CREATE ROLE :"role" WITH LOGIN PASSWORD :'"'"'pw'"'"' CREATEDB SUPERUSER;'log "User $PGUSER created"@@ -566,9 +586,9 @@ cmd_setup_native() {fi# Step 4: Create portos database if it doesn't exist- if ! psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw "$PGDATABASE"; then+ if ! pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw "$PGDATABASE"; theninfo "Creating database: $PGDATABASE"- psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -c "CREATE DATABASE $PGDATABASE OWNER $PGUSER;"+ pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d postgres -c "CREATE DATABASE $PGDATABASE OWNER $PGUSER;"log "Database $PGDATABASE created"elselog "Database $PGDATABASE already exists"@@ -577,9 +597,9 @@ cmd_setup_native() {# Step 5: Enable pgvector extension and apply schemainfo "Applying schema..."# pgvector extension requires superuser — create as system user, then run schema as portos- psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS vector;" 2>/dev/null || true- psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS pgcrypto;" 2>/dev/null || true- PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -v ON_ERROR_STOP=1 --single-transaction -f "$ROOT_DIR/server/scripts/init-db.sql"+ pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS vector;" 2>/dev/null || true+ pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$sys_user" -d "$PGDATABASE" -c "CREATE EXTENSION IF NOT EXISTS pgcrypto;" 2>/dev/null || true+ PGPASSWORD="$PGPASSWORD" pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -v ON_ERROR_STOP=1 --single-transaction -f "$ROOT_DIR/server/scripts/init-db.sql"log "Schema applied"echo ""@@ -612,7 +632,7 @@ run_psql() {return $?fiif command -v psql >/dev/null 2>&1; then- PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@"+ PGPASSWORD="$PGPASSWORD" pg_bound psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@"elif uses_local_docker_endpoint && docker_running; thenPGPASSWORD="$PGPASSWORD" docker exec -i -e PGPASSWORD portos-db psql -U "$PGUSER" -d "$PGDATABASE" "$@"else@@ -630,7 +650,7 @@ run_pg_dump() {if uses_local_docker_endpoint && docker_running; thenPGPASSWORD="$PGPASSWORD" docker exec -e PGPASSWORD portos-db pg_dump -U "$PGUSER" -d "$PGDATABASE" "$@"elif command -v pg_dump >/dev/null 2>&1; then- PGPASSWORD="$PGPASSWORD" pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@"+ PGPASSWORD="$PGPASSWORD" pg_bound pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@"elseerr "pg_dump not found on host and Docker DB is not running" >&2# return (not exit) so cmd_export can clean up its temp filediff --git a/scripts/setup-db.js b/scripts/setup-db.jsindex c5e2e88c40..a8f6f45d49 100644--- a/scripts/setup-db.js+++ b/scripts/setup-db.js@@ -44,8 +44,19 @@ const PG_PORT_DOCKER = parseDockerPort(envVar('PGPORT_DOCKER', 5561));// child process. Without this, db.sh setup-native would provision the// default `portos`/`portos` while isPortOSDbReady() probes with the// customized creds — leaving setup looping forever.+//+// Inherited libpq routing variables are dropped: PGHOSTADDR overrides the+// network destination of the explicit PGHOST, PGSERVICE/PGSERVICEFILE can name+// another endpoint, and PGOPTIONS alters session settings. A shell left over+// from another PostgreSQL workflow must not redirect the readiness probe or+// role provisioning to a different cluster. TLS/auth variables stay. Keep in+// step with PG_ROUTING_VARS in db.sh.+const ROUTING_ENV = ['PGHOSTADDR', 'PGSERVICE', 'PGSERVICEFILE', 'PGOPTIONS'];+const inheritedEnv = Object.fromEntries(+ Object.entries(process.env).filter(([name]) => !ROUTING_ENV.includes(name))+);const PG_CHILD_ENV = {- ...process.env,+ ...inheritedEnv,PGUSER: PG_USER,PGDATABASE: PG_DATABASE,PGPASSWORD: PG_PASSWORD,diff --git a/scripts/setup-db.test.js b/scripts/setup-db.test.jsindex 563d3f8625..804ff7ac0f 100644--- a/scripts/setup-db.test.js+++ b/scripts/setup-db.test.js@@ -66,7 +66,7 @@ describe('native setup inherited endpoint', () => {// Run the actual CLI body with synthetic configuration and subprocesses. No// imports execute, no install .env is read, and no database can be contacted.-async function runSetup({ mode = 'docker', unavailable, tty = false, nativeReady = true, running = true, hostBinding = '127.0.0.1', hostPort = 5561, dotEnv = {}, exportedMode } = {}) {+async function runSetup({ mode = 'docker', unavailable, tty = false, nativeReady = true, running = true, hostBinding = '127.0.0.1', hostPort = 5561, dotEnv = {}, exportedMode, exportedEnv = {} } = {}) {const savedEnv = { PGMODE: mode, EXAMPLE_SETTING: 'preserved', ...dotEnv };const initialEnv = { ...savedEnv };const calls = [];@@ -91,7 +91,7 @@ async function runSetup({ mode = 'docker', unavailable, tty = false, nativeReadycreateInterface: () => { throw new Error('Setup must not prompt to switch backends'); },resolveBashBinary: () => 'bash',process: {- env: exportedMode === undefined ? {} : { PGMODE: exportedMode }, platform: 'linux', stdin: { isTTY: tty }, stdout: { isTTY: tty },+ env: { ...(exportedMode === undefined ? {} : { PGMODE: exportedMode }), ...exportedEnv }, platform: 'linux', stdin: { isTTY: tty }, stdout: { isTTY: tty },exit: (code) => { exitCode = code; throw exitSignal; }},console: { log: (message) => logs.push(message), error: (message) => errors.push(message) },@@ -224,6 +224,28 @@ describe('setup preserves the selected database', () => {expect(result.calls.some((call) => call.includes('setup-native'))).toBe(!nativeReady);});+ // The command-boundary contract for #10757: a shell left over from another+ // PostgreSQL workflow must not redirect the readiness probe or the bootstrap+ // child; TLS/auth settings keep working.+ it('drops inherited libpq routing variables from the readiness probe and bootstrap child', async () => {+ const result = await runSetup({+ mode: 'native', nativeReady: false,+ dotEnv: { PGHOST: 'db.example.invalid', PGPORT: '5433' },+ exportedEnv: {+ PGHOSTADDR: '192.0.2.10', PGSERVICE: 'other', PGSERVICEFILE: '/example/service.conf',+ PGOPTIONS: '-c search_path=other', PGSSLMODE: 'verify-full', PGPASSFILE: '/example/pgpass',+ },+ });+ expect(result.exitCode).toBe(0);+ const psqlEnvs = result.childEnvs.filter((_, i) => result.calls[i][0] === 'psql');+ const bootstrapEnv = result.childEnvs[result.calls.findIndex((call) => call.includes('setup-native'))];+ expect(psqlEnvs).toHaveLength(2);+ for (const env of [...psqlEnvs, bootstrapEnv]) {+ for (const name of ['PGHOSTADDR', 'PGSERVICE', 'PGSERVICEFILE', 'PGOPTIONS']) expect(env).not.toHaveProperty(name);+ expect(env).toMatchObject({ PGHOST: 'db.example.invalid', PGPORT: '5433', PGSSLMODE: 'verify-full', PGPASSFILE: '/example/pgpass' });+ }+ });+it('probes and provisions the endpoint selected in .env, host and port alike', async () => {const result = await runSetup({mode: 'native', nativeReady: false,