Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop database repair when shutdown or lock cleanup fails (#11253)

Adam Eivycc6d8804ecbe
diff --git a/scripts/db-repair.test.js b/scripts/db-repair.test.js
new file mode 100644
index 0000000000..e321bdebf4
--- /dev/null
+++ b/scripts/db-repair.test.js
@@ -0,0 +1,156 @@
+// Exercise the real repair/start dispatch with synthetic files and Docker
+// executables. No daemon, PostgreSQL endpoint or real data volume is accessed.
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { spawnSync } from 'node:child_process';
+import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = dirname(fileURLToPath(import.meta.url));
+const image = readFileSync(join(here, '..', 'docker-compose.yml'), 'utf8').match(/image: (\S+)/)[1];
+
+function executable(path, content) {
+ writeFileSync(path, content);
+ chmodSync(path, 0o755);
+}
+
+describe.skipIf(process.platform === 'win32')('database Docker repair safety', () => {
+ let root;
+ let lock;
+ let logFile;
+
+ beforeEach(() => {
+ root = mkdtempSync(join(tmpdir(), 'portos-db-repair-'));
+ mkdirSync(join(root, 'scripts', 'lib'), { recursive: true });
+ mkdirSync(join(root, 'bin'));
+ mkdirSync(join(root, 'volume'));
+ copyFileSync(join(here, 'db.sh'), join(root, 'scripts', 'db.sh'));
+ copyFileSync(join(here, 'lib', 'envFile.cjs'), join(root, 'scripts', 'lib', 'envFile.cjs'));
+ copyFileSync(join(here, '..', 'docker-compose.yml'), join(root, 'docker-compose.yml'));
+ symlinkSync(process.execPath, join(root, 'bin', 'node'));
+ executable(join(root, 'bin', 'uname'), '#!/bin/sh\necho Linux\n');
+ executable(join(root, 'bin', 'sleep'), '#!/bin/sh\nexit 0\n');
+ executable(join(root, 'bin', 'docker'), `#!/bin/bash
+set -eu
+echo "docker $*" >> "$STUB_LOG"
+case "$1 \${2:-}" in
+ 'info '|'compose version') exit 0 ;;
+ 'ps -aq')
+ [ "\${STUB_LIST_FAIL:-0}" = 0 ] || exit 20
+ if [ ! -f "$STUB_ROOT/removed" ] && [ "\${STUB_ABSENT:-0}" = 0 ]; then echo fixture-db; fi ;;
+ 'inspect -f')
+ if [[ "$3" == *State.Running* ]]; then
+ [ "\${STUB_STATE_FAIL:-0}" = 0 ] || exit 21
+ echo "\${STUB_RUNNING:-false}"
+ else echo fixture-recorded-volume; fi ;;
+ 'compose stop') exit "\${STUB_STOP_RC:-0}" ;;
+ 'rm fixture-db')
+ [ "\${STUB_RM_RC:-0}" = 0 ] || exit "$STUB_RM_RC"
+ touch "$STUB_ROOT/removed" ;;
+ 'volume inspect') exit "\${STUB_VOLUME_RC:-0}" ;;
+ 'compose config')
+ if [ "$3" = --images ]; then echo "$STUB_IMAGE";
+ else echo '{"services":{"db":{"volumes":[{"type":"volume","source":"db-data","target":"/var/lib/postgresql/data"}]}},"volumes":{"db-data":{"name":"fixture-configured-volume"}}}'; fi ;;
+ 'run --rm')
+ [ "\${STUB_HELPER_RC:-0}" = 0 ] || exit "$STUB_HELPER_RC"
+ # Execute the actual helper command against the fixture directory. A no-op
+ # rm simulates a lock that survives; the helper must verify its absence.
+ command="\${!#}"
+ command="\${command//\\/data/$STUB_ROOT/volume}"
+ if [ "\${STUB_LOCK_PERSISTS:-0}" = 1 ]; then
+ function rm() { return 0; }
+ export -f rm
+ fi
+ bash -c "$command" ;;
+ 'compose up') touch "$STUB_ROOT/started" ;;
+ 'compose exec')
+ if [ "$5" = pg_isready ]; then
+ [ -f "$STUB_ROOT/removed" ] && [ "\${STUB_NEVER_READY:-0}" = 0 ] || exit 1
+ else echo 1; fi ;;
+ 'logs portos-db') echo 'bogus data in lock file' ;;
+ *) echo 'Unexpected Docker call' >&2; exit 99 ;;
+esac
+`);
+ lock = join(root, 'volume', 'postmaster.pid');
+ logFile = join(root, 'docker.log');
+ writeFileSync(lock, 'synthetic lock');
+ writeFileSync(logFile, '');
+ });
+
+ afterEach(() => rmSync(root, { recursive: true, force: true }));
+
+ const run = (overrides = {}, action = 'fix') => spawnSync('bash', [join(root, 'scripts', 'db.sh'), action], {
+ encoding: 'utf8',
+ timeout: 10000,
+ env: {
+ PATH: `${join(root, 'bin')}:/usr/bin:/bin`,
+ HOME: root,
+ PGMODE: 'docker',
+ STUB_ROOT: root,
+ STUB_LOG: logFile,
+ STUB_IMAGE: image,
+ ...overrides,
+ },
+ });
+ const log = () => readFileSync(logFile, 'utf8');
+
+ it.each([
+ ['failed shutdown', { STUB_STOP_RC: '17', STUB_RUNNING: 'true' }],
+ ['failed removal', { STUB_RM_RC: '18' }],
+ ['container still running after a successful stop', { STUB_RUNNING: 'true' }],
+ ['unanswered state inspection', { STUB_STATE_FAIL: '1' }],
+ ['unanswered container listing', { STUB_LIST_FAIL: '1' }],
+ ['unverified volume', { STUB_VOLUME_RC: '1' }],
+ ])('refuses lock deletion after %s', (_, overrides) => {
+ const result = run(overrides);
+ expect(result.status, result.stderr).not.toBe(0);
+ expect(existsSync(lock)).toBe(true);
+ expect(log()).not.toMatch(/^docker run /m);
+ expect(result.stdout).not.toContain('Stale lock files cleaned');
+ });
+
+ it.each([
+ ['helper command failure', { STUB_HELPER_RC: '19' }],
+ ['lock remains after removal', { STUB_LOCK_PERSISTS: '1' }],
+ ])('propagates %s without reporting cleanup success', (_, overrides) => {
+ const result = run(overrides);
+ expect(result.status, result.stderr).not.toBe(0);
+ expect(existsSync(lock)).toBe(true);
+ expect(log()).toMatch(/^docker run /m);
+ expect(result.stdout).not.toContain('Stale lock files cleaned');
+ });
+
+ it.each([
+ ['stopped container', {}, 'fixture-recorded-volume'],
+ ['absent container', { STUB_ABSENT: '1' }, 'fixture-configured-volume'],
+ ])('repairs a verified %s using its exact volume and cached pinned image', (_, overrides, volume) => {
+ const result = run(overrides);
+ expect(result.status, result.stderr).toBe(0);
+ expect(existsSync(lock)).toBe(false);
+ expect(log()).toContain(`docker run --rm --pull=never -v ${volume}:/data ${image} sh -c`);
+ expect(result.stdout).toContain('Stale lock files cleaned');
+ if (overrides.STUB_ABSENT) expect(log()).not.toMatch(/^docker (rm|compose stop) /m);
+ });
+
+ it('automatically repairs once and retries startup successfully', () => {
+ const result = run({}, 'start');
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain('PostgreSQL ready');
+ expect(log().match(/^docker run /gm)).toHaveLength(1);
+ expect(log().match(/^docker compose up /gm)).toHaveLength(2);
+ });
+
+ it('does not restart after failed automatic repair or repair again after a failed retry', () => {
+ const failedRepair = run({ STUB_STOP_RC: '17' }, 'start');
+ expect(failedRepair.status, failedRepair.stderr).not.toBe(0);
+ expect(log().match(/^docker compose up /gm)).toHaveLength(1);
+ expect(log()).not.toMatch(/^docker run /m);
+
+ writeFileSync(logFile, '');
+ const failedRetry = run({ STUB_NEVER_READY: '1' }, 'start');
+ expect(failedRetry.status, failedRetry.stderr).not.toBe(0);
+ expect(log().match(/^docker compose up /gm)).toHaveLength(2);
+ expect(log().match(/^docker run /gm)).toHaveLength(1);
+ });
+});
diff --git a/scripts/db.sh b/scripts/db.sh
index b435f181a5..0bc4268081 100755
--- a/scripts/db.sh
+++ b/scripts/db.sh
@@ -471,13 +471,60 @@ fix_docker() {
cd "$ROOT_DIR"
- # Determine the actual data volume used by the portos-db container, if it exists
- local data_volume=""
- data_volume=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Destination "/var/lib/postgresql/data" }}{{ .Name }}{{ end }}{{ end }}' portos-db 2>/dev/null || echo "")
-
- # Stop and remove container
- docker compose stop db 2>/dev/null || true
- docker rm -f portos-db 2>/dev/null || true
+ # A successful list distinguishes an absent container from an unavailable
+ # daemon. Never turn a failed inspection into permission to modify its volume.
+ local container_id data_volume running
+ if ! container_id=$(docker ps -aq --filter 'name=^/portos-db$'); then
+ err "Could not establish the database container state; repair stopped"
+ return 1
+ fi
+ if [ -n "$container_id" ]; then
+ # Retain the actual volume before removing the container, not a guessed
+ # Compose project name (which may have changed since it was provisioned).
+ if ! data_volume=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Destination "/var/lib/postgresql/data" }}{{ .Name }}{{ end }}{{ end }}' "$container_id"); then
+ err "Could not resolve the database data volume; repair stopped"
+ return 1
+ fi
+ if ! docker compose stop db; then
+ err "Failed to stop Docker PostgreSQL; repair stopped"
+ return 1
+ fi
+ if ! running=$(docker inspect -f '{{.State.Running}}' "$container_id") || [ "$running" != false ]; then
+ err "Database container is still running or its state is unknown; repair stopped"
+ return 1
+ fi
+ # Do not force-remove a container that may have restarted since inspection.
+ if ! docker rm "$container_id"; then
+ err "Failed to remove the database container; repair stopped"
+ return 1
+ fi
+ else
+ # No container exists. Ask Compose for the exact configured volume name;
+ # never try guessed names, which docker run would silently create.
+ if ! data_volume=$(docker compose config --format json | node -e '
+ let input = "";
+ process.stdin.on("data", chunk => { input += chunk; });
+ process.stdin.on("end", () => {
+ const config = JSON.parse(input);
+ const mount = config.services?.db?.volumes?.find(v => v.type === "volume" && v.target === "/var/lib/postgresql/data");
+ const name = config.volumes?.[mount?.source]?.name;
+ if (typeof name !== "string" || !name) process.exit(1);
+ process.stdout.write(name);
+ });
+ '); then
+ err "Could not resolve the configured database volume; repair stopped"
+ return 1
+ fi
+ fi
+ # Verify absence after removal too, including any concurrent restart.
+ if ! container_id=$(docker ps -aq --filter 'name=^/portos-db$') || [ -n "$container_id" ]; then
+ err "Database container absence could not be verified; repair stopped"
+ return 1
+ fi
+ if [ -z "$data_volume" ] || ! docker volume inspect "$data_volume" >/dev/null; then
+ err "Database data volume could not be verified; repair stopped"
+ return 1
+ fi
# Remove stale postmaster.pid from the volume. Reuse the pinned, already-pulled
# db image from docker-compose.yml (has rm/cat) so repair works offline.
@@ -488,14 +535,10 @@ fix_docker() {
warn "Could not resolve the db image from docker-compose.yml — cannot remove stale lock files"
return 1
fi
- if [ -n "$data_volume" ]; then
- docker run --rm -v "${data_volume}:/data" "$PG_HELPER_IMAGE" rm -f /data/postmaster.pid 2>/dev/null || true
- else
- # Fallback: derive volume name from compose project name
- local project_name
- project_name=$(docker compose config --format json 2>/dev/null | grep -o '"name":"[^"]*"' | head -1 | cut -d'"' -f4 || echo "portos")
- docker run --rm -v "${project_name}_portos-pgdata:/data" "$PG_HELPER_IMAGE" rm -f /data/postmaster.pid 2>/dev/null ||
- docker run --rm -v "portos-pgdata:/data" "$PG_HELPER_IMAGE" rm -f /data/postmaster.pid 2>/dev/null || true
+ if ! docker run --rm --pull=never -v "${data_volume}:/data" "$PG_HELPER_IMAGE" \
+ sh -c 'rm -f /data/postmaster.pid && test ! -e /data/postmaster.pid'; then
+ err "Failed to remove or verify the stale database lock; repair stopped"
+ return 1
fi
log "Stale lock files cleaned"