A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(installer): stop upgrades hanging after the uploads copy (#3478)
Hampus812919759ace
diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.ymlindex bfbaba678..cf8e3622c 100644--- a/deploy/self-hosting/docker-compose.yml+++ b/deploy/self-hosting/docker-compose.yml@@ -215,7 +215,7 @@ x-fluxer-app-healthcheck: &fluxer-app-healthcheckx-fluxer-svc-healthcheck: &fluxer-svc-healthcheck<<: *fluxer-app-healthcheckstart_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s}- test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]+ test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8090 && printf \"GET /_health HTTP/1.0\\r\\n\\r\\n\" >&3 && head -n 1 <&3 | grep -q \" 200 \"'"]services:edge:@@ -469,7 +469,7 @@ services:<<: *fluxer-envFLUXER_API_PORT: "8080"FLUXER_SETUP_TOKEN: ${FLUXER_SETUP_TOKEN:-}- NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_API_NODE_HEAP_MB}${FLUXER_API_NODE_OPTIONS:+ $FLUXER_API_NODE_OPTIONS}+ NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=${FLUXER_API_NODE_HEAP_MB:-}}${FLUXER_API_NODE_OPTIONS:+ ${FLUXER_API_NODE_OPTIONS:-}}NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt}FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_API_POSTGRES_MAX_CONNECTIONS:-25}"FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: "${FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED:-true}"@@ -503,13 +503,13 @@ services:command: ["node", "dist/WorkerEntrypoint.js"]environment:<<: *fluxer-env- NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB}${FLUXER_WORKER_NODE_OPTIONS:+ $FLUXER_WORKER_NODE_OPTIONS}+ NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=${FLUXER_WORKER_NODE_HEAP_MB:-}}${FLUXER_WORKER_NODE_OPTIONS:+ ${FLUXER_WORKER_NODE_OPTIONS:-}}NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt}FLUXER_API_WORKER_MODE: all_lanesFLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: "true"FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS:-25}"healthcheck:- test: ["CMD", "node", "-e", "const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}"]+ test: ["CMD-SHELL", "node -e \"const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}\""]<<: *fluxer-app-healthcheckretries: ${FLUXER_WORKER_HEALTHCHECK_RETRIES:-3}depends_on:@@ -867,7 +867,7 @@ services:FLUXER_API_ENDPOINT: http://api:8080FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}healthcheck:- test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]+ test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf \"GET /_health HTTP/1.0\\r\\n\\r\\n\" >&3 && head -n 1 <&3 | grep -q \" 200 \"'"]<<: *fluxer-app-healthcheckstart_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s}depends_on:diff --git a/fluxer_docs/src/installer/install.ps1 b/fluxer_docs/src/installer/install.ps1index 08b87e949..233b5fb68 100644--- a/fluxer_docs/src/installer/install.ps1+++ b/fluxer_docs/src/installer/install.ps1@@ -1454,6 +1454,17 @@ function Get-FluxerFreeKb([string]$Path) {return [long]($drive.AvailableFreeSpace / 1024)}+function Start-FluxerStopped([string[]]$Ids) {+ if ($Ids.Count -eq 0) {+ return 0+ }+ $result = Invoke-FluxerCapture (@('start') + $Ids)+ if ($result.Code -ne 0 -and $result.Error.Length -gt 0) {+ Write-FluxerProblem $result.Error+ }+ return $result.Code+}+# Step 2 continued: the volume copy.## A live copy can catch a file mid-write, so the stack stops for it. The stack comes back up on@@ -1461,9 +1472,10 @@ function Get-FluxerFreeKb([string]$Path) {# working instance rather than a stopped one.## By hand:+# docker compose ps -q > running# docker compose stop# docker run --rm -v fluxer_seaweedfs-data:/data -v "${PWD}\backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .-# docker compose up -d+# docker start (Get-Content running)function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDir) {if (-not (Test-FluxerStackDefinesService 'seaweedfs' $TargetDir)) {Write-FluxerLine 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects are outside the stack and only the operator of that store can copy them.'@@ -1497,6 +1509,11 @@ function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDi$tarExtension = 'tar'}Write-FluxerLine 'Stopping the stack for a consistent copy of the uploads.'+ $running = Invoke-FluxerCapture @('compose', 'ps', '-q')+ if ($running.Code -ne 0) {+ Stop-Fluxer "docker compose ps failed, so the containers to start again after the copy cannot be read. Compose printed:`n$(Get-FluxerIndented $running.Error ' ')" $FluxerExitBackup+ }+ $stopped = @($running.Text.Split("`n") | ForEach-Object {$_.Trim()} | Where-Object {$_.Length -gt 0})if ((Invoke-FluxerDocker @('compose', 'stop')) -ne 0) {Stop-Fluxer 'docker compose stop failed.' $FluxerExitBackup}@@ -1505,13 +1522,13 @@ function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDiWrite-FluxerLine "Copying $full."$code = Invoke-FluxerDocker @('run', '--rm', '-v', "${full}:/data:ro", '-v', "${Record}:/backup", $FluxerHelperImage, 'tar', $tarFlags, "/backup/$volume.$tarExtension", '-C', '/data', '.')if ($code -ne 0) {- [void](Invoke-FluxerDocker @('compose', 'up', '-d', '--remove-orphans'))+ [void](Start-FluxerStopped $stopped)Stop-Fluxer "Copying $full failed. The stack is started again on the images it was running." $FluxerExitBackup}}Write-FluxerLine 'Starting the stack again before the upgrade continues.'- if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--remove-orphans')) -ne 0) {- Stop-Fluxer 'docker compose up -d failed after the copy. Read docker compose logs.' $FluxerExitBackup+ if ((Start-FluxerStopped $stopped) -ne 0) {+ Stop-Fluxer 'docker start failed after the copy. Read docker compose logs.' $FluxerExitBackup}}diff --git a/fluxer_docs/src/installer/install.sh b/fluxer_docs/src/installer/install.shindex 3d0dddcc5..b0b404823 100644--- a/fluxer_docs/src/installer/install.sh+++ b/fluxer_docs/src/installer/install.sh@@ -1686,6 +1686,11 @@ fluxer_free_kb() {df -Pk "$1" 2>/dev/null | awk 'NR==2 {print $4}'}+fluxer_start_stopped() {+ [ -s "$fluxer_scratch/stopped" ] || return 0+ xargs $fluxer_engine start < "$fluxer_scratch/stopped" > /dev/null+}+# Step 2 continued: the volume copy.## A live copy can catch a file mid-write, so the stack stops for it. The stack@@ -1693,9 +1698,10 @@ fluxer_free_kb() {# happens, so a failure here leaves a working instance rather than a stopped one.## By hand:+# docker compose ps -q > running# docker compose stop# docker run --rm -v fluxer_seaweedfs-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .-# docker compose up -d+# xargs docker start < runningfluxer_copy_volumes() {if ! fluxer_stack_defines_service seaweedfs; thenfluxer_say 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects are outside the stack and only the operator of that store can copy them.'@@ -1738,6 +1744,9 @@ $(fluxer_volume_error ' ')" ;;fluxer_tar_ext='tgz'fifluxer_say 'Stopping the stack for a consistent copy of the uploads.'+ if ! $fluxer_engine compose ps -q > "$fluxer_scratch/stopped"; then+ fluxer_fail 7 "$fluxer_engine compose ps failed in $opt_dir, so the containers to start again after the copy cannot be read."+ fiif ! $fluxer_engine compose stop; thenfluxer_fail 7 "$fluxer_engine compose stop failed in $opt_dir."fi@@ -1746,13 +1755,13 @@ $(fluxer_volume_error ' ')" ;;fluxer_full="${fluxer_project}_${fluxer_volume}"fluxer_say "Copying $fluxer_full."if ! $fluxer_engine run --rm -v "$fluxer_full:/data:ro" -v "$fluxer_record:/backup" "$FLUXER_HELPER_IMAGE" tar "$fluxer_tar_flags" "/backup/$fluxer_volume.$fluxer_tar_ext" -C /data .; then- $fluxer_engine compose up -d --remove-orphans || true+ fluxer_start_stopped || truefluxer_fail 7 "Copying $fluxer_full failed. The stack is started again on the images it was running."fidone < "$fluxer_scratch/copy-volumes"fluxer_say 'Starting the stack again before the upgrade continues.'- if ! $fluxer_engine compose up -d --remove-orphans; then- fluxer_fail 7 "$fluxer_engine compose up -d failed in $opt_dir after the copy. Read $fluxer_engine compose logs there."+ if ! fluxer_start_stopped; then+ fluxer_fail 7 "$fluxer_engine start failed in $opt_dir after the copy. Read $fluxer_engine compose logs there."fi}