Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: forward resolved database settings to Docker Compose instead of letting it reparse .env (#10759)
Adam Eivyb902c431eab7
diff --git a/.env.example b/.env.exampleindex 71cbf629d5..846d0584e0 100644--- a/.env.example+++ b/.env.example@@ -8,6 +8,11 @@# PostgreSQL password (must match docker-compose.yml POSTGRES_PASSWORD in Docker mode)# Note: Changing this after initial setup requires manually updating the PostgreSQL user password (or recreating the DB volume).+# Values are read literally (surrounding quotes stripped; no inline comments or $ expansion).+# `npm run setup:db` and scripts/db.sh pass the resolved values to Docker Compose, so the+# container is provisioned with exactly what PortOS connects with. If you run `docker compose`+# directly, Compose applies its OWN grammar to this file: a `#` after whitespace starts a+# comment and `$` expands. Wrap a value containing either in single quotes, e.g. PGPASSWORD='pa$$word'.PGPASSWORD=portos# Optional overrides (defaults auto-detect from PGMODE)diff --git a/docs/STORAGE.md b/docs/STORAGE.mdindex 7973041cf7..8b2aa0e0cd 100644--- a/docs/STORAGE.md+++ b/docs/STORAGE.md@@ -410,6 +410,8 @@ The escape hatch is **guarded from bitrot by the test suite** (tests boot with ``PGPASSWORD`/`PGUSER`/`PGDATABASE`/`PGHOST`/`PGPORT` are resolved from `process.env` first, then `.env`, then the backward-compatible defaults (`portos`/`portos`/`portos`/`5432`). The default `portos` password is an **intentional** local-development fallback (see the Distribution model note in [`AGENTS.md`](../AGENTS.md)); production deployments override it via `PGPASSWORD`.+**Compose receives the resolved values.** PortOS and PM2 read `.env` literally (matching surrounding quotes removed; no inline comments, no `$` expansion), but `docker-compose.yml` interpolates its `PGUSER`/`PGDATABASE`/`PGPASSWORD`/`PGPORT_DOCKER` with Docker's own `.env` grammar, which treats ` #` as a comment and expands `$`. So `npm run setup:db` and `scripts/db.sh` put the already-resolved values in the environment of every `docker compose` subprocess (process environment outranks Compose's `.env`), and the container is provisioned with exactly what PortOS connects with. They travel in the child environment only — never argv or logs. If you run `docker compose` yourself, Compose applies its own grammar to `.env`: wrap a password containing ` #` or `$` in single quotes (`PGPASSWORD='pa$$word'` — in Compose, single quotes are literal), or export the variables first.+### Moving between Docker and nativeUse **Settings → Database** for coordinated backend migration, includingdiff --git a/scripts/db-migrate.test.js b/scripts/db-migrate.test.jsindex bbc4469135..27b30afa07 100644--- a/scripts/db-migrate.test.js+++ b/scripts/db-migrate.test.js@@ -22,6 +22,7 @@ case "$1" inps) echo "Up 5 minutes"; exit 0 ;;info) exit 0 ;;compose)+ [ -n "$COMPOSE_ENV_LOG" ] && echo "$PGUSER|$PGDATABASE|$PGPASSWORD|$PGPORT_DOCKER" >> "$COMPOSE_ENV_LOG"case "$*" in *" stop "*) echo SOURCE_STOP >> "$STUB_LOG" ;; esacexit 0 ;;exec)@@ -448,6 +449,31 @@ exit "\${PROVISION_EXIT:-0}"expect(readFileSync(join(root, 'psql.env'), 'utf8')).toBe('role"x|' + SECRET);});+ it('forwards the resolved settings to Compose so its own .env grammar cannot change them', () => {+ const composeLog = join(root, 'compose-env.log');+ writeFileSync(envFile, [+ 'PGMODE=docker',+ 'PGUSER=example-user',+ 'PGDATABASE=example_db',+ 'PGPASSWORD=example-pass # local note',+ 'PGPORT_DOCKER=5599',+ '',+ ].join('\n'));+ const result = run(['stop'], 'ok', { PGPASSWORD: '', COMPOSE_ENV_LOG: composeLog });+ expect(result.status, result.stderr).toBe(0);+ const forwarded = () => new Set(readFileSync(composeLog, 'utf8').trim().split('\n'));+ expect(forwarded()).toEqual(new Set(['example-user|example_db|example-pass # local note|5599']));+ expect(readFileSync(stubLog, 'utf8')).not.toContain('example-pass');+ expect(result.stdout + result.stderr).not.toContain('example-pass');++ // A literal dollar expression is forwarded verbatim, not expanded.+ writeFileSync(envFile, 'PGMODE=docker\nPGPASSWORD="example-$PORTOS_PROBE_SUFFIX"\n');+ writeFileSync(composeLog, '');+ const dollar = run(['stop'], 'ok', { PGPASSWORD: '', COMPOSE_ENV_LOG: composeLog });+ expect(dollar.status, dollar.stderr).toBe(0);+ expect(forwarded()).toEqual(new Set(['portos|portos|example-$PORTOS_PROBE_SUFFIX|5561']));+ });+it('setup-native fails when role provisioning fails', () => {writeStub(join(root, 'bin'), 'psql', PROVISION_PSQL_STUB);const result = run(['setup-native'], 'ok', { PGPASSWORD: SECRET, STUB_LOG_DIR: root, PROVISION_EXIT: '3' });diff --git a/scripts/db.sh b/scripts/db.shindex d4d44a39fd..39a75d4bda 100755--- a/scripts/db.sh+++ b/scripts/db.sh@@ -69,6 +69,12 @@ if [ "$_DB_CONFIG_COMPLETE" != true ]; thenexit 1fiunset key value _DB_CONFIG_COMPLETE+# Compose interpolates docker-compose.yml with its own .env grammar (inline+# comments, `$` expansion), which differs from the literal parser above. Process+# environment outranks Compose's .env, so exporting the resolved values makes+# every `docker compose` call below provision exactly what PortOS connects with.+# Exported to child environments only — never argv or logs.+export PGUSER PGDATABASE PGPASSWORD PGPORT_DOCKER# The maintenance coordinator names its install's dump directory explicitly# (its data root can differ from this checkout); otherwise use this checkout.DUMP_DIR="${PORTOS_DUMP_DIR:-$ROOT_DIR/data/db-dumps}"diff --git a/scripts/setup-db.js b/scripts/setup-db.jsindex 2337f16352..c5e2e88c40 100644--- a/scripts/setup-db.js+++ b/scripts/setup-db.js@@ -53,6 +53,20 @@ const PG_CHILD_ENV = {PGPORT: String(PG_PORT_NATIVE)};+// Environment for every `docker compose` subprocess. Compose interpolates+// docker-compose.yml from its own .env grammar (inline comments, `$` expansion),+// which differs from the literal parser PortOS and PM2 share. Process+// environment outranks Compose's .env, so forwarding the values resolved above+// makes the container provision exactly what PortOS connects with. Child+// environment only — the password never reaches argv or logs.+const COMPOSE_ENV = {+ ...process.env,+ PGUSER: PG_USER,+ PGDATABASE: PG_DATABASE,+ PGPASSWORD: PG_PASSWORD,+ PGPORT_DOCKER: String(PG_PORT_DOCKER)+};+function getMode() {// Nonempty exported PGMODE → .env → docker; must match ecosystem.config.cjs (#10758).return process.env.PGMODE || envFile.PGMODE || 'docker';@@ -140,7 +154,7 @@ function isDockerSchemaReady() {'docker',['compose', 'exec', '-T', 'db', 'psql', '-X', '-U', PG_USER, '-d', PG_DATABASE, '-tAc',"SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'memories' LIMIT 1"],- { stdio: 'pipe', cwd: rootDir }+ { stdio: 'pipe', cwd: rootDir, env: COMPOSE_ENV }).toString();return output.trim() === '1';} catch {@@ -163,7 +177,8 @@ function waitForHealth(maxAttempts = 30) {try {execFileSync('docker', ['compose', 'exec', '-T', 'db', 'pg_isready', '-h', '127.0.0.1', '-U', PG_USER], {stdio: 'pipe',- cwd: rootDir+ cwd: rootDir,+ env: COMPOSE_ENV});if (isDockerSchemaReady()) return true;} catch {@@ -300,7 +315,8 @@ console.log('🐳 Reconciling PostgreSQL container configuration...');try {execFileSync('docker', ['compose', 'up', '-d', 'db'], {stdio: 'inherit',- cwd: rootDir+ cwd: rootDir,+ env: COMPOSE_ENV});} catch (err) {console.error(`❌ Failed to reconcile PostgreSQL container: ${err.message}`);diff --git a/scripts/setup-db.test.js b/scripts/setup-db.test.jsindex 4a3f6643a6..563d3f8625 100644--- a/scripts/setup-db.test.js+++ b/scripts/setup-db.test.js@@ -1,5 +1,7 @@import { describe, it, expect } from 'vitest';-import { readFileSync } from 'fs';+import { spawnSync } from 'node:child_process';+import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'fs';+import { tmpdir } from 'node:os';import { runInNewContext } from 'node:vm';import { dirname, join } from 'path';import { fileURLToPath } from 'url';@@ -111,8 +113,8 @@ async function runSetup({ mode = 'docker', unavailable, tty = false, nativeReadyreturn '';}if (invocation.startsWith('docker compose exec -T db psql')) return '1\n';- if (['docker --version', 'docker info', 'docker compose version',- 'docker compose exec -T db pg_isready -h 127.0.0.1 -U portos'].includes(invocation)) return '';+ if (['docker --version', 'docker info', 'docker compose version'].includes(invocation)+ || invocation.startsWith('docker compose exec -T db pg_isready -h 127.0.0.1 -U ')) return '';throw new Error(`Unexpected synthetic subprocess: ${invocation}`);}});@@ -153,6 +155,32 @@ describe('setup preserves the selected database', () => {});+ it('hands Compose the resolved settings on every compose subprocess, not its own .env parse', async () => {+ const result = await runSetup({+ dotEnv: {+ PGUSER: 'example-user',+ PGDATABASE: 'example_db',+ PGPASSWORD: 'example-pass # local note',+ PGPORT_DOCKER: '5599'+ }+ });+ expect(result.exitCode).toBe(0);+ const composeEnvs = result.calls+ .map((call, i) => (call[0] === 'docker' && call[1] === 'compose' && !['version'].includes(call[2]) ? result.childEnvs[i] : null))+ .filter(Boolean);+ expect(composeEnvs.length).toBeGreaterThanOrEqual(3); // up, pg_isready, psql+ for (const env of composeEnvs) {+ expect(env).toMatchObject({+ PGUSER: 'example-user',+ PGDATABASE: 'example_db',+ PGPASSWORD: 'example-pass # local note',+ PGPORT_DOCKER: '5599'+ });+ }+ expect(result.calls.flat().join(' ')).not.toContain('example-pass');+ expect(result.logs.concat(result.errors).join('\n')).not.toContain('example-pass');+ });+it.each([true, false])('succeeds with selected Docker (container running: %s)', async (running) => {const result = await runSetup({ running });expect(result.exitCode).toBe(0);@@ -223,3 +251,45 @@ describe('setup preserves the selected database', () => {expect(result.calls.some(([command]) => command === 'docker')).toBe(!expectNative);});});++// Config-only: `docker compose config` never contacts the daemon or mutates a+// container. Skipped when the Compose plugin is not installed.+const composeAvailable = spawnSync('docker', ['compose', 'version'], { stdio: 'ignore' }).status === 0;+describe.skipIf(!composeAvailable)('Compose receives the resolved settings (config only)', () => {+ it('process environment overrides the .env interpolation of comments and dollar expressions', () => {+ const dir = mkdtempSync(join(tmpdir(), 'portos-compose-'));+ try {+ writeFileSync(join(dir, 'docker-compose.yml'), dockerComposeSrc);+ writeFileSync(join(dir, '.env'), [+ 'PGUSER=example-user',+ 'PGDATABASE=example_db',+ 'PGPASSWORD="example-$PORTOS_PROBE_SUFFIX"',+ 'PGPORT_DOCKER=5599 # note',+ '',+ ].join('\n'));+ const config = (extraEnv) => {+ const { PGUSER, PGDATABASE, PGPASSWORD, PGPORT_DOCKER, ...base } = process.env;+ const result = spawnSync('docker', ['compose', 'config', '--format', 'json'], {+ cwd: dir, encoding: 'utf8', env: { ...base, ...extraEnv }+ });+ expect(result.status, result.stderr).toBe(0);+ return JSON.parse(result.stdout).services.db;+ };+ // Control: Compose alone expands the dollar expression — the divergence being fixed.+ expect(config({}).environment.POSTGRES_PASSWORD).toBe('example-');+ const forwarded = config({+ PGUSER: 'example-user', PGDATABASE: 'example_db',+ PGPASSWORD: 'example-$PORTOS_PROBE_SUFFIX', PGPORT_DOCKER: '5599'+ });+ expect(forwarded.environment).toMatchObject({+ POSTGRES_USER: 'example-user',+ POSTGRES_DB: 'example_db',+ // `config` re-escapes a literal `$` as `$$`; the container receives one `$`.+ POSTGRES_PASSWORD: 'example-$$PORTOS_PROBE_SUFFIX'+ });+ expect(forwarded.ports[0].published).toBe('5599');+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+});