Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

feat(self-hosting): run postgres or the object store outside (#2620)

Hampus1b22d14f3d5a
diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.example
index 298884654..ec85e094f 100644
--- a/deploy/self-hosting/.env.example
+++ b/deploy/self-hosting/.env.example
@@ -117,6 +117,21 @@ FLUXER_IMAGE_TAG=v1
POSTGRES_PASSWORD=CHANGE_ME
MEILI_MASTER_KEY=CHANGE_ME
+# The stack ships its own Postgres and its own object store, and points at both
+# by service name. Set these to run either one outside the stack. Leave them
+# unset and the bundled services are used. Taking a service out of the stack
+# means an upgrade skips the backup step that reaches into it, and backing that
+# store up belongs to whoever runs it.
+#FLUXER_POSTGRES_HOST=db.example.com
+#FLUXER_POSTGRES_PORT=5432
+#FLUXER_POSTGRES_DATABASE=fluxer
+#FLUXER_POSTGRES_USERNAME=fluxer
+#FLUXER_POSTGRES_SSL=true
+#FLUXER_S3_ENDPOINT=https://s3.eu-central-1.amazonaws.com
+#FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com
+#FLUXER_S3_REGION=eu-central-1
+#FLUXER_S3_FORCE_PATH_STYLE=false
+
FLUXER_S3_ACCESS_KEY=fluxer
FLUXER_S3_SECRET_KEY=CHANGE_ME
diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml
index 66498cdf8..437ba7999 100644
--- a/deploy/self-hosting/docker-compose.yml
+++ b/deploy/self-hosting/docker-compose.yml
@@ -2,12 +2,12 @@ name: fluxer
x-fluxer-postgres-env: &fluxer-postgres-env
FLUXER_DATABASE_BACKEND: postgres
- FLUXER_POSTGRES_HOST: postgres
- FLUXER_POSTGRES_PORT: "5432"
- FLUXER_POSTGRES_DATABASE: fluxer
- FLUXER_POSTGRES_USERNAME: fluxer
+ FLUXER_POSTGRES_HOST: ${FLUXER_POSTGRES_HOST:-postgres}
+ FLUXER_POSTGRES_PORT: "${FLUXER_POSTGRES_PORT:-5432}"
+ FLUXER_POSTGRES_DATABASE: ${FLUXER_POSTGRES_DATABASE:-fluxer}
+ FLUXER_POSTGRES_USERNAME: ${FLUXER_POSTGRES_USERNAME:-fluxer}
FLUXER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
- FLUXER_POSTGRES_SSL: "false"
+ FLUXER_POSTGRES_SSL: "${FLUXER_POSTGRES_SSL:-false}"
FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-true}
x-fluxer-env: &fluxer-env
@@ -35,12 +35,12 @@ x-fluxer-env: &fluxer-env
FLUXER_SEARCH_URL: http://meilisearch:7700
FLUXER_SEARCH_API_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env}
- FLUXER_S3_ENDPOINT: http://seaweedfs:8333
- FLUXER_S3_PUBLIC_ENDPOINT: http://seaweedfs:8333
- FLUXER_S3_REGION: us-east-1
+ FLUXER_S3_ENDPOINT: ${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}
+ FLUXER_S3_PUBLIC_ENDPOINT: ${FLUXER_S3_PUBLIC_ENDPOINT:-${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}}
+ FLUXER_S3_REGION: ${FLUXER_S3_REGION:-us-east-1}
FLUXER_S3_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env}
FLUXER_S3_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env}
- FLUXER_S3_FORCE_PATH_STYLE: "true"
+ FLUXER_S3_FORCE_PATH_STYLE: "${FLUXER_S3_FORCE_PATH_STYLE:-true}"
FLUXER_S3_BUCKET_CDN: fluxer
FLUXER_S3_BUCKET_UPLOADS: fluxer-uploads
FLUXER_S3_BUCKET_DOWNLOADS: fluxer-downloads
diff --git a/fluxer_docs/src/content/docs/operator/upgrading.mdx b/fluxer_docs/src/content/docs/operator/upgrading.mdx
index 68a14658a..03a10e512 100644
--- a/fluxer_docs/src/content/docs/operator/upgrading.mdx
+++ b/fluxer_docs/src/content/docs/operator/upgrading.mdx
@@ -212,6 +212,25 @@ Put a dump on a timer as well. On Linux and macOS, this crontab line writes one
`/srv/fluxer` stands for the directory holding `.env`. Write it as an absolute path, because cron does not expand `~`. An unescaped `%` in a crontab is a newline, which is why every one of them has a backslash.
+## Run a data store outside the stack
+
+The stack ships its own Postgres and its own object store and points at both by service name. `.env` moves either one somewhere else, and the bundled service is used when the line is absent:
+
+```ini
+FLUXER_POSTGRES_HOST=db.example.com
+FLUXER_POSTGRES_PORT=5432
+FLUXER_POSTGRES_SSL=true
+FLUXER_S3_ENDPOINT=https://s3.eu-central-1.amazonaws.com
+FLUXER_S3_REGION=eu-central-1
+FLUXER_S3_FORCE_PATH_STYLE=false
+```
+
+`FLUXER_S3_PUBLIC_ENDPOINT` follows `FLUXER_S3_ENDPOINT` when it is not set on its own, and the credentials stay `FLUXER_S3_ACCESS_KEY` and `FLUXER_S3_SECRET_KEY`.
+
+Pointing the stack elsewhere leaves the bundled service defined and running with nothing reading it. Take it out with an override file listed in `COMPOSE_FILE`, which [Keep a local compose change](#keep-a-local-compose-change) describes, rather than by editing `docker-compose.yml`, which the Place step replaces on every upgrade.
+
+An upgrade backs up what the stack holds. The Dump step reaches into the `postgres` service and the Copy step reaches into the `seaweedfs` volume, so a stack that defines neither has neither step to run. The run says which one it skipped and goes on. Backing up a store outside the stack belongs to whoever runs it, and the record the upgrade writes holds no dump of it, so take one before upgrading if a rollback would need it.
+
## Roll back
A rollback puts the previous release back:
diff --git a/fluxer_docs/src/installer/install.ps1 b/fluxer_docs/src/installer/install.ps1
index 4cc0ecbdf..6a6ee864e 100644
--- a/fluxer_docs/src/installer/install.ps1
+++ b/fluxer_docs/src/installer/install.ps1
@@ -1281,7 +1281,28 @@ function Test-FluxerDumpHeader([string]$Path) {
# The volume copy measures its volume and refuses when the disk is short. This step does not,
# because the size of a custom-format dump is not knowable before pg_dump writes it. Point
# -BackupDir at a drive with room for the database.
+# The bundled data stores are services in the stack file. An operator who points the stack at a
+# database or an object store outside it takes those services out, and the two backup steps that
+# reach into them then have nothing to reach. That is a supported shape rather than a fault, so
+# each step says what it skipped and the upgrade goes on. Backing up a store outside the stack
+# belongs to whoever runs it.
+$script:FluxerStackServices = $null
+
+function Test-FluxerStackDefinesService([string]$Name, [string]$TargetDir) {
+ if ($null -eq $script:FluxerStackServices) {
+ if ((Invoke-FluxerComposeQuery '--services') -ne 0) {
+ Stop-Fluxer "docker compose config --services failed in $TargetDir, so the services this stack defines cannot be read. Compose printed:`n$(Get-FluxerComposeError ' ')" $FluxerExitUnhealthy
+ }
+ $script:FluxerStackServices = @(Get-FluxerComposeServices)
+ }
+ return $script:FluxerStackServices -contains $Name
+}
+
function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) {
+ if (-not (Test-FluxerStackDefinesService 'postgres' $TargetDir)) {
+ Write-FluxerLine 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.'
+ return
+ }
if (-not (Test-FluxerPostgresRunning)) {
Write-FluxerLine 'Postgres is not running. Starting it for the dump.'
if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--wait', 'postgres')) -ne 0) {
@@ -1340,7 +1361,11 @@ function Get-FluxerFreeKb([string]$Path) {
# docker compose stop
# docker run --rm -v fluxer_seaweedfs-data:/data -v "${PWD}\backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .
# docker compose up -d
-function Copy-FluxerVolumes([string]$Record, [string]$Project) {
+function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDir) {
+ if (-not (Test-FluxerStackDefinesService 'seaweedfs' $TargetDir)) {
+ Write-FluxerLine 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects live outside the stack and only the operator of that store can copy them.'
+ return
+ }
$present = @()
foreach ($volume in $FluxerBackupVolumes) {
$full = "${Project}_$volume"
@@ -1391,7 +1416,7 @@ function Backup-FluxerInstance([string]$Record, [string]$TargetDir, [string]$Pro
Write-FluxerLine 'Skipping the uploads copy. -NoVolumeBackup was given.'
return
}
- Copy-FluxerVolumes $Record $Project
+ Copy-FluxerVolumes $Record $Project $TargetDir
}
# A newer Postgres major does not read the data directory an older major wrote, so moving between
diff --git a/fluxer_docs/src/installer/install.sh b/fluxer_docs/src/installer/install.sh
index b1bb4602a..c43f0e478 100644
--- a/fluxer_docs/src/installer/install.sh
+++ b/fluxer_docs/src/installer/install.sh
@@ -1351,7 +1351,28 @@ fluxer_postgres_running() {
# step does not, because the size of a custom-format dump is not knowable before
# pg_dump writes it. Point --backup-dir at a filesystem with room for the
# database.
+# The bundled data stores are services in the stack file. An operator who points
+# the stack at a database or an object store outside it takes those services out,
+# and the two backup steps that reach into them then have nothing to reach. That
+# is a supported shape rather than a fault, so each step says what it skipped and
+# the upgrade goes on. Backing up a store outside the stack belongs to whoever
+# runs it.
+fluxer_stack_defines_service() {
+ if [ ! -f "$fluxer_scratch/all-services" ]; then
+ if ! fluxer_compose_services > "$fluxer_scratch/all-services"; then
+ rm -f "$fluxer_scratch/all-services"
+ fluxer_fail 6 "docker compose config --services failed in $opt_dir, so the services this stack defines cannot be read. Compose printed:
+$(fluxer_compose_error ' ')"
+ fi
+ fi
+ grep -qxF "$1" "$fluxer_scratch/all-services"
+}
+
fluxer_dump_postgres() {
+ if ! fluxer_stack_defines_service postgres; then
+ fluxer_say 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.'
+ return 0
+ fi
if ! fluxer_postgres_running; then
fluxer_say 'Postgres is not running. Starting it for the dump.'
if ! $fluxer_engine compose up -d --wait postgres; then
@@ -1400,6 +1421,10 @@ fluxer_free_kb() {
# docker run --rm -v fluxer_seaweedfs-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .
# docker compose up -d
fluxer_copy_volumes() {
+ if ! fluxer_stack_defines_service seaweedfs; then
+ fluxer_say 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects live outside the stack and only the operator of that store can copy them.'
+ return 0
+ fi
fluxer_backup_volumes > "$fluxer_scratch/backup-volumes"
: > "$fluxer_scratch/copy-volumes"
fluxer_copy_any=0