A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(self-hosting): run postgres or the object store outside (#2620)
Hampus1b22d14f3d5a
diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.exampleindex 298884654..ec85e094f 100644--- a/deploy/self-hosting/.env.example+++ b/deploy/self-hosting/.env.example@@ -117,6 +117,21 @@ FLUXER_IMAGE_TAG=v1POSTGRES_PASSWORD=CHANGE_MEMEILI_MASTER_KEY=CHANGE_ME+# The stack ships its own Postgres and its own object store, and points at both+# by service name. Set these to run either one outside the stack. Leave them+# unset and the bundled services are used. Taking a service out of the stack+# means an upgrade skips the backup step that reaches into it, and backing that+# store up belongs to whoever runs it.+#FLUXER_POSTGRES_HOST=db.example.com+#FLUXER_POSTGRES_PORT=5432+#FLUXER_POSTGRES_DATABASE=fluxer+#FLUXER_POSTGRES_USERNAME=fluxer+#FLUXER_POSTGRES_SSL=true+#FLUXER_S3_ENDPOINT=https://s3.eu-central-1.amazonaws.com+#FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com+#FLUXER_S3_REGION=eu-central-1+#FLUXER_S3_FORCE_PATH_STYLE=false+FLUXER_S3_ACCESS_KEY=fluxerFLUXER_S3_SECRET_KEY=CHANGE_MEdiff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.ymlindex 66498cdf8..437ba7999 100644--- a/deploy/self-hosting/docker-compose.yml+++ b/deploy/self-hosting/docker-compose.yml@@ -2,12 +2,12 @@ name: fluxerx-fluxer-postgres-env: &fluxer-postgres-envFLUXER_DATABASE_BACKEND: postgres- FLUXER_POSTGRES_HOST: postgres- FLUXER_POSTGRES_PORT: "5432"- FLUXER_POSTGRES_DATABASE: fluxer- FLUXER_POSTGRES_USERNAME: fluxer+ FLUXER_POSTGRES_HOST: ${FLUXER_POSTGRES_HOST:-postgres}+ FLUXER_POSTGRES_PORT: "${FLUXER_POSTGRES_PORT:-5432}"+ FLUXER_POSTGRES_DATABASE: ${FLUXER_POSTGRES_DATABASE:-fluxer}+ FLUXER_POSTGRES_USERNAME: ${FLUXER_POSTGRES_USERNAME:-fluxer}FLUXER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}- FLUXER_POSTGRES_SSL: "false"+ FLUXER_POSTGRES_SSL: "${FLUXER_POSTGRES_SSL:-false}"FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-true}x-fluxer-env: &fluxer-env@@ -35,12 +35,12 @@ x-fluxer-env: &fluxer-envFLUXER_SEARCH_URL: http://meilisearch:7700FLUXER_SEARCH_API_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env}- FLUXER_S3_ENDPOINT: http://seaweedfs:8333- FLUXER_S3_PUBLIC_ENDPOINT: http://seaweedfs:8333- FLUXER_S3_REGION: us-east-1+ FLUXER_S3_ENDPOINT: ${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}+ FLUXER_S3_PUBLIC_ENDPOINT: ${FLUXER_S3_PUBLIC_ENDPOINT:-${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}}+ FLUXER_S3_REGION: ${FLUXER_S3_REGION:-us-east-1}FLUXER_S3_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env}FLUXER_S3_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env}- FLUXER_S3_FORCE_PATH_STYLE: "true"+ FLUXER_S3_FORCE_PATH_STYLE: "${FLUXER_S3_FORCE_PATH_STYLE:-true}"FLUXER_S3_BUCKET_CDN: fluxerFLUXER_S3_BUCKET_UPLOADS: fluxer-uploadsFLUXER_S3_BUCKET_DOWNLOADS: fluxer-downloadsdiff --git a/fluxer_docs/src/content/docs/operator/upgrading.mdx b/fluxer_docs/src/content/docs/operator/upgrading.mdxindex 68a14658a..03a10e512 100644--- a/fluxer_docs/src/content/docs/operator/upgrading.mdx+++ b/fluxer_docs/src/content/docs/operator/upgrading.mdx@@ -212,6 +212,25 @@ Put a dump on a timer as well. On Linux and macOS, this crontab line writes one`/srv/fluxer` stands for the directory holding `.env`. Write it as an absolute path, because cron does not expand `~`. An unescaped `%` in a crontab is a newline, which is why every one of them has a backslash.+## Run a data store outside the stack++The stack ships its own Postgres and its own object store and points at both by service name. `.env` moves either one somewhere else, and the bundled service is used when the line is absent:++```ini+FLUXER_POSTGRES_HOST=db.example.com+FLUXER_POSTGRES_PORT=5432+FLUXER_POSTGRES_SSL=true+FLUXER_S3_ENDPOINT=https://s3.eu-central-1.amazonaws.com+FLUXER_S3_REGION=eu-central-1+FLUXER_S3_FORCE_PATH_STYLE=false+```++`FLUXER_S3_PUBLIC_ENDPOINT` follows `FLUXER_S3_ENDPOINT` when it is not set on its own, and the credentials stay `FLUXER_S3_ACCESS_KEY` and `FLUXER_S3_SECRET_KEY`.++Pointing the stack elsewhere leaves the bundled service defined and running with nothing reading it. Take it out with an override file listed in `COMPOSE_FILE`, which [Keep a local compose change](#keep-a-local-compose-change) describes, rather than by editing `docker-compose.yml`, which the Place step replaces on every upgrade.++An upgrade backs up what the stack holds. The Dump step reaches into the `postgres` service and the Copy step reaches into the `seaweedfs` volume, so a stack that defines neither has neither step to run. The run says which one it skipped and goes on. Backing up a store outside the stack belongs to whoever runs it, and the record the upgrade writes holds no dump of it, so take one before upgrading if a rollback would need it.+## Roll backA rollback puts the previous release back:diff --git a/fluxer_docs/src/installer/install.ps1 b/fluxer_docs/src/installer/install.ps1index 4cc0ecbdf..6a6ee864e 100644--- a/fluxer_docs/src/installer/install.ps1+++ b/fluxer_docs/src/installer/install.ps1@@ -1281,7 +1281,28 @@ function Test-FluxerDumpHeader([string]$Path) {# The volume copy measures its volume and refuses when the disk is short. This step does not,# because the size of a custom-format dump is not knowable before pg_dump writes it. Point# -BackupDir at a drive with room for the database.+# The bundled data stores are services in the stack file. An operator who points the stack at a+# database or an object store outside it takes those services out, and the two backup steps that+# reach into them then have nothing to reach. That is a supported shape rather than a fault, so+# each step says what it skipped and the upgrade goes on. Backing up a store outside the stack+# belongs to whoever runs it.+$script:FluxerStackServices = $null++function Test-FluxerStackDefinesService([string]$Name, [string]$TargetDir) {+ if ($null -eq $script:FluxerStackServices) {+ if ((Invoke-FluxerComposeQuery '--services') -ne 0) {+ Stop-Fluxer "docker compose config --services failed in $TargetDir, so the services this stack defines cannot be read. Compose printed:`n$(Get-FluxerComposeError ' ')" $FluxerExitUnhealthy+ }+ $script:FluxerStackServices = @(Get-FluxerComposeServices)+ }+ return $script:FluxerStackServices -contains $Name+}+function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) {+ if (-not (Test-FluxerStackDefinesService 'postgres' $TargetDir)) {+ Write-FluxerLine 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.'+ return+ }if (-not (Test-FluxerPostgresRunning)) {Write-FluxerLine 'Postgres is not running. Starting it for the dump.'if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--wait', 'postgres')) -ne 0) {@@ -1340,7 +1361,11 @@ function Get-FluxerFreeKb([string]$Path) {# docker compose stop# docker run --rm -v fluxer_seaweedfs-data:/data -v "${PWD}\backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .# docker compose up -d-function Copy-FluxerVolumes([string]$Record, [string]$Project) {+function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDir) {+ if (-not (Test-FluxerStackDefinesService 'seaweedfs' $TargetDir)) {+ Write-FluxerLine 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects live outside the stack and only the operator of that store can copy them.'+ return+ }$present = @()foreach ($volume in $FluxerBackupVolumes) {$full = "${Project}_$volume"@@ -1391,7 +1416,7 @@ function Backup-FluxerInstance([string]$Record, [string]$TargetDir, [string]$ProWrite-FluxerLine 'Skipping the uploads copy. -NoVolumeBackup was given.'return}- Copy-FluxerVolumes $Record $Project+ Copy-FluxerVolumes $Record $Project $TargetDir}# A newer Postgres major does not read the data directory an older major wrote, so moving betweendiff --git a/fluxer_docs/src/installer/install.sh b/fluxer_docs/src/installer/install.shindex b1bb4602a..c43f0e478 100644--- a/fluxer_docs/src/installer/install.sh+++ b/fluxer_docs/src/installer/install.sh@@ -1351,7 +1351,28 @@ fluxer_postgres_running() {# step does not, because the size of a custom-format dump is not knowable before# pg_dump writes it. Point --backup-dir at a filesystem with room for the# database.+# The bundled data stores are services in the stack file. An operator who points+# the stack at a database or an object store outside it takes those services out,+# and the two backup steps that reach into them then have nothing to reach. That+# is a supported shape rather than a fault, so each step says what it skipped and+# the upgrade goes on. Backing up a store outside the stack belongs to whoever+# runs it.+fluxer_stack_defines_service() {+ if [ ! -f "$fluxer_scratch/all-services" ]; then+ if ! fluxer_compose_services > "$fluxer_scratch/all-services"; then+ rm -f "$fluxer_scratch/all-services"+ fluxer_fail 6 "docker compose config --services failed in $opt_dir, so the services this stack defines cannot be read. Compose printed:+$(fluxer_compose_error ' ')"+ fi+ fi+ grep -qxF "$1" "$fluxer_scratch/all-services"+}+fluxer_dump_postgres() {+ if ! fluxer_stack_defines_service postgres; then+ fluxer_say 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.'+ return 0+ fiif ! fluxer_postgres_running; thenfluxer_say 'Postgres is not running. Starting it for the dump.'if ! $fluxer_engine compose up -d --wait postgres; then@@ -1400,6 +1421,10 @@ fluxer_free_kb() {# docker run --rm -v fluxer_seaweedfs-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .# docker compose up -dfluxer_copy_volumes() {+ if ! fluxer_stack_defines_service seaweedfs; then+ fluxer_say 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects live outside the stack and only the operator of that store can copy them.'+ return 0+ fifluxer_backup_volumes > "$fluxer_scratch/backup-volumes": > "$fluxer_scratch/copy-volumes"fluxer_copy_any=0