Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop database repair when shutdown or lock cleanup fails (#11253)
Adam Eivycc6d8804ecbe
diff --git a/scripts/db-repair.test.js b/scripts/db-repair.test.jsnew file mode 100644index 0000000000..e321bdebf4--- /dev/null+++ b/scripts/db-repair.test.js@@ -0,0 +1,156 @@+// Exercise the real repair/start dispatch with synthetic files and Docker+// executables. No daemon, PostgreSQL endpoint or real data volume is accessed.+import { afterEach, beforeEach, describe, expect, it } from 'vitest';+import { spawnSync } from 'node:child_process';+import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { fileURLToPath } from 'node:url';++const here = dirname(fileURLToPath(import.meta.url));+const image = readFileSync(join(here, '..', 'docker-compose.yml'), 'utf8').match(/image: (\S+)/)[1];++function executable(path, content) {+ writeFileSync(path, content);+ chmodSync(path, 0o755);+}++describe.skipIf(process.platform === 'win32')('database Docker repair safety', () => {+ let root;+ let lock;+ let logFile;++ beforeEach(() => {+ root = mkdtempSync(join(tmpdir(), 'portos-db-repair-'));+ mkdirSync(join(root, 'scripts', 'lib'), { recursive: true });+ mkdirSync(join(root, 'bin'));+ mkdirSync(join(root, 'volume'));+ copyFileSync(join(here, 'db.sh'), join(root, 'scripts', 'db.sh'));+ copyFileSync(join(here, 'lib', 'envFile.cjs'), join(root, 'scripts', 'lib', 'envFile.cjs'));+ copyFileSync(join(here, '..', 'docker-compose.yml'), join(root, 'docker-compose.yml'));+ symlinkSync(process.execPath, join(root, 'bin', 'node'));+ executable(join(root, 'bin', 'uname'), '#!/bin/sh\necho Linux\n');+ executable(join(root, 'bin', 'sleep'), '#!/bin/sh\nexit 0\n');+ executable(join(root, 'bin', 'docker'), `#!/bin/bash+set -eu+echo "docker $*" >> "$STUB_LOG"+case "$1 \${2:-}" in+ 'info '|'compose version') exit 0 ;;+ 'ps -aq')+ [ "\${STUB_LIST_FAIL:-0}" = 0 ] || exit 20+ if [ ! -f "$STUB_ROOT/removed" ] && [ "\${STUB_ABSENT:-0}" = 0 ]; then echo fixture-db; fi ;;+ 'inspect -f')+ if [[ "$3" == *State.Running* ]]; then+ [ "\${STUB_STATE_FAIL:-0}" = 0 ] || exit 21+ echo "\${STUB_RUNNING:-false}"+ else echo fixture-recorded-volume; fi ;;+ 'compose stop') exit "\${STUB_STOP_RC:-0}" ;;+ 'rm fixture-db')+ [ "\${STUB_RM_RC:-0}" = 0 ] || exit "$STUB_RM_RC"+ touch "$STUB_ROOT/removed" ;;+ 'volume inspect') exit "\${STUB_VOLUME_RC:-0}" ;;+ 'compose config')+ if [ "$3" = --images ]; then echo "$STUB_IMAGE";+ else echo '{"services":{"db":{"volumes":[{"type":"volume","source":"db-data","target":"/var/lib/postgresql/data"}]}},"volumes":{"db-data":{"name":"fixture-configured-volume"}}}'; fi ;;+ 'run --rm')+ [ "\${STUB_HELPER_RC:-0}" = 0 ] || exit "$STUB_HELPER_RC"+ # Execute the actual helper command against the fixture directory. A no-op+ # rm simulates a lock that survives; the helper must verify its absence.+ command="\${!#}"+ command="\${command//\\/data/$STUB_ROOT/volume}"+ if [ "\${STUB_LOCK_PERSISTS:-0}" = 1 ]; then+ function rm() { return 0; }+ export -f rm+ fi+ bash -c "$command" ;;+ 'compose up') touch "$STUB_ROOT/started" ;;+ 'compose exec')+ if [ "$5" = pg_isready ]; then+ [ -f "$STUB_ROOT/removed" ] && [ "\${STUB_NEVER_READY:-0}" = 0 ] || exit 1+ else echo 1; fi ;;+ 'logs portos-db') echo 'bogus data in lock file' ;;+ *) echo 'Unexpected Docker call' >&2; exit 99 ;;+esac+`);+ lock = join(root, 'volume', 'postmaster.pid');+ logFile = join(root, 'docker.log');+ writeFileSync(lock, 'synthetic lock');+ writeFileSync(logFile, '');+ });++ afterEach(() => rmSync(root, { recursive: true, force: true }));++ const run = (overrides = {}, action = 'fix') => spawnSync('bash', [join(root, 'scripts', 'db.sh'), action], {+ encoding: 'utf8',+ timeout: 10000,+ env: {+ PATH: `${join(root, 'bin')}:/usr/bin:/bin`,+ HOME: root,+ PGMODE: 'docker',+ STUB_ROOT: root,+ STUB_LOG: logFile,+ STUB_IMAGE: image,+ ...overrides,+ },+ });+ const log = () => readFileSync(logFile, 'utf8');++ it.each([+ ['failed shutdown', { STUB_STOP_RC: '17', STUB_RUNNING: 'true' }],+ ['failed removal', { STUB_RM_RC: '18' }],+ ['container still running after a successful stop', { STUB_RUNNING: 'true' }],+ ['unanswered state inspection', { STUB_STATE_FAIL: '1' }],+ ['unanswered container listing', { STUB_LIST_FAIL: '1' }],+ ['unverified volume', { STUB_VOLUME_RC: '1' }],+ ])('refuses lock deletion after %s', (_, overrides) => {+ const result = run(overrides);+ expect(result.status, result.stderr).not.toBe(0);+ expect(existsSync(lock)).toBe(true);+ expect(log()).not.toMatch(/^docker run /m);+ expect(result.stdout).not.toContain('Stale lock files cleaned');+ });++ it.each([+ ['helper command failure', { STUB_HELPER_RC: '19' }],+ ['lock remains after removal', { STUB_LOCK_PERSISTS: '1' }],+ ])('propagates %s without reporting cleanup success', (_, overrides) => {+ const result = run(overrides);+ expect(result.status, result.stderr).not.toBe(0);+ expect(existsSync(lock)).toBe(true);+ expect(log()).toMatch(/^docker run /m);+ expect(result.stdout).not.toContain('Stale lock files cleaned');+ });++ it.each([+ ['stopped container', {}, 'fixture-recorded-volume'],+ ['absent container', { STUB_ABSENT: '1' }, 'fixture-configured-volume'],+ ])('repairs a verified %s using its exact volume and cached pinned image', (_, overrides, volume) => {+ const result = run(overrides);+ expect(result.status, result.stderr).toBe(0);+ expect(existsSync(lock)).toBe(false);+ expect(log()).toContain(`docker run --rm --pull=never -v ${volume}:/data ${image} sh -c`);+ expect(result.stdout).toContain('Stale lock files cleaned');+ if (overrides.STUB_ABSENT) expect(log()).not.toMatch(/^docker (rm|compose stop) /m);+ });++ it('automatically repairs once and retries startup successfully', () => {+ const result = run({}, 'start');+ expect(result.status, result.stderr).toBe(0);+ expect(result.stdout).toContain('PostgreSQL ready');+ expect(log().match(/^docker run /gm)).toHaveLength(1);+ expect(log().match(/^docker compose up /gm)).toHaveLength(2);+ });++ it('does not restart after failed automatic repair or repair again after a failed retry', () => {+ const failedRepair = run({ STUB_STOP_RC: '17' }, 'start');+ expect(failedRepair.status, failedRepair.stderr).not.toBe(0);+ expect(log().match(/^docker compose up /gm)).toHaveLength(1);+ expect(log()).not.toMatch(/^docker run /m);++ writeFileSync(logFile, '');+ const failedRetry = run({ STUB_NEVER_READY: '1' }, 'start');+ expect(failedRetry.status, failedRetry.stderr).not.toBe(0);+ expect(log().match(/^docker compose up /gm)).toHaveLength(2);+ expect(log().match(/^docker run /gm)).toHaveLength(1);+ });+});diff --git a/scripts/db.sh b/scripts/db.shindex b435f181a5..0bc4268081 100755--- a/scripts/db.sh+++ b/scripts/db.sh@@ -471,13 +471,60 @@ fix_docker() {cd "$ROOT_DIR"- # Determine the actual data volume used by the portos-db container, if it exists- local data_volume=""- data_volume=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Destination "/var/lib/postgresql/data" }}{{ .Name }}{{ end }}{{ end }}' portos-db 2>/dev/null || echo "")-- # Stop and remove container- docker compose stop db 2>/dev/null || true- docker rm -f portos-db 2>/dev/null || true+ # A successful list distinguishes an absent container from an unavailable+ # daemon. Never turn a failed inspection into permission to modify its volume.+ local container_id data_volume running+ if ! container_id=$(docker ps -aq --filter 'name=^/portos-db$'); then+ err "Could not establish the database container state; repair stopped"+ return 1+ fi+ if [ -n "$container_id" ]; then+ # Retain the actual volume before removing the container, not a guessed+ # Compose project name (which may have changed since it was provisioned).+ if ! data_volume=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Destination "/var/lib/postgresql/data" }}{{ .Name }}{{ end }}{{ end }}' "$container_id"); then+ err "Could not resolve the database data volume; repair stopped"+ return 1+ fi+ if ! docker compose stop db; then+ err "Failed to stop Docker PostgreSQL; repair stopped"+ return 1+ fi+ if ! running=$(docker inspect -f '{{.State.Running}}' "$container_id") || [ "$running" != false ]; then+ err "Database container is still running or its state is unknown; repair stopped"+ return 1+ fi+ # Do not force-remove a container that may have restarted since inspection.+ if ! docker rm "$container_id"; then+ err "Failed to remove the database container; repair stopped"+ return 1+ fi+ else+ # No container exists. Ask Compose for the exact configured volume name;+ # never try guessed names, which docker run would silently create.+ if ! data_volume=$(docker compose config --format json | node -e '+ let input = "";+ process.stdin.on("data", chunk => { input += chunk; });+ process.stdin.on("end", () => {+ const config = JSON.parse(input);+ const mount = config.services?.db?.volumes?.find(v => v.type === "volume" && v.target === "/var/lib/postgresql/data");+ const name = config.volumes?.[mount?.source]?.name;+ if (typeof name !== "string" || !name) process.exit(1);+ process.stdout.write(name);+ });+ '); then+ err "Could not resolve the configured database volume; repair stopped"+ return 1+ fi+ fi+ # Verify absence after removal too, including any concurrent restart.+ if ! container_id=$(docker ps -aq --filter 'name=^/portos-db$') || [ -n "$container_id" ]; then+ err "Database container absence could not be verified; repair stopped"+ return 1+ fi+ if [ -z "$data_volume" ] || ! docker volume inspect "$data_volume" >/dev/null; then+ err "Database data volume could not be verified; repair stopped"+ return 1+ fi# Remove stale postmaster.pid from the volume. Reuse the pinned, already-pulled# db image from docker-compose.yml (has rm/cat) so repair works offline.@@ -488,14 +535,10 @@ fix_docker() {warn "Could not resolve the db image from docker-compose.yml — cannot remove stale lock files"return 1fi- if [ -n "$data_volume" ]; then- docker run --rm -v "${data_volume}:/data" "$PG_HELPER_IMAGE" rm -f /data/postmaster.pid 2>/dev/null || true- else- # Fallback: derive volume name from compose project name- local project_name- project_name=$(docker compose config --format json 2>/dev/null | grep -o '"name":"[^"]*"' | head -1 | cut -d'"' -f4 || echo "portos")- docker run --rm -v "${project_name}_portos-pgdata:/data" "$PG_HELPER_IMAGE" rm -f /data/postmaster.pid 2>/dev/null ||- docker run --rm -v "portos-pgdata:/data" "$PG_HELPER_IMAGE" rm -f /data/postmaster.pid 2>/dev/null || true+ if ! docker run --rm --pull=never -v "${data_volume}:/data" "$PG_HELPER_IMAGE" \+ sh -c 'rm -f /data/postmaster.pid && test ! -e /data/postmaster.pid'; then+ err "Failed to remove or verify the stale database lock; repair stopped"+ return 1filog "Stale lock files cleaned"