Skip to content

Open-source manufacturing ERP, MES and QMS. Quoting, MRP, inventory, shop floor, quality and lot/serial traceability on one Postgres schema, with a REST API and MCP server. Self-host or use Carbon Cloud.

16.2 KB
Raw Blame
#!/usr/bin/env bash
#
# Carbon — single-VPS Docker Swarm deploy helper.
#
# Wraps the Swarm stack: bootstraps secrets, builds the app images, deploys the
# stack, and runs DB migrations in the correct order. See README.md.
#
# Usage:
#   ./deploy.sh init [--force]    # swarm init + create Docker secrets + .env
#   ./deploy.sh build             # build erp+mes images on this host
#   ./deploy.sh deploy            # docker stack deploy
#   ./deploy.sh up                # build + deploy + wait + migrate (one shot)
#   ./deploy.sh migrate           # (re)apply DB migrations only
#   ./deploy.sh secret <name> <value>   # create/rotate one secret
#   ./deploy.sh status            # services + tasks
#   ./deploy.sh logs <service>    # follow a service's logs (e.g. erp, caddy)
#   ./deploy.sh down [--volumes]  # remove the stack (--volumes also wipes data)
#
set -euo pipefail

readonly SCRIPT_NAME=$(basename "$0")
readonly HERE=$(cd "$(dirname "$0")" && pwd)
readonly STACK_FILE="$HERE/docker-compose.prod.yml"
readonly ENV_FILE="$HERE/.env"
readonly ENV_EXAMPLE="$HERE/.env.example"

# Secrets generated by `init` (value auto-created) vs. supplied by the operator.
# postgres_password is handled on its own (paired with postgrest_db_uri), so it is
# NOT in GENERATED_SECRETS. ALL_SECRETS is the full set the stack needs to deploy.
readonly GENERATED_SECRETS="session_secret inngest_signing_key inngest_event_key"
readonly TRIO_SECRETS="jwt_secret anon_key service_role_key"
readonly SUPPLIED_SECRETS="resend_api_key smtp_password saml_private_key"
readonly ALL_SECRETS="postgres_password postgrest_db_uri $GENERATED_SECRETS $TRIO_SECRETS $SUPPLIED_SECRETS"

log()   { printf '\033[0;36m[%s]\033[0m %s\n' "$(date '+%H:%M:%S')" "$*"; }
warn()  { printf '\033[0;33m[warn]\033[0m %s\n' "$*" >&2; }
error() { printf '\033[0;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }

require_cmds() {
    local missing=()
    for c in "$@"; do command -v "$c" >/dev/null 2>&1 || missing+=("$c"); done
    [ ${#missing[@]} -eq 0 ] || error "missing required command(s): ${missing[*]}"
}

load_env() {
    [ -f "$ENV_FILE" ] || error ".env not found — run '$SCRIPT_NAME init' first"
    set -a; . "$ENV_FILE"; set +a
    : "${CARBON_REPO:?CARBON_REPO must be set in .env}"
    : "${STACK_NAME:?STACK_NAME must be set in .env}"
    # Defaulted here so a .env written before this image existed still works.
    : "${CARBON_IMAGE_BOOTSTRAP:=carbon/bootstrap:latest}"
    [ -f "$CARBON_REPO/Dockerfile" ] || error "CARBON_REPO=$CARBON_REPO is not a Carbon checkout (no Dockerfile)"
}

swarm_active() { [ "$(docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null)" = "active" ]; }
secret_exists() { docker secret inspect "$1" >/dev/null 2>&1; }
create_secret() { printf '%s' "$2" | docker secret create "$1" - >/dev/null && log "secret '$1' created"; }

# Create or replace a secret. Refuses (clear error) if it is in use by the running
# stack — Swarm secrets are immutable while attached, so rotation needs `down` first.
replace_secret() {
    if secret_exists "$1"; then
        docker secret rm "$1" >/dev/null 2>&1 \
            || error "secret '$1' is in use — run '$SCRIPT_NAME down' first, then retry"
    fi
    create_secret "$1" "$2"
}

# postgres_password and postgrest_db_uri MUST always carry the same password. The
# PostgREST image is FROM scratch (no shell) so it can't run the secrets-entrypoint
# shim to assemble the URI from the password at boot — it reads a ready-made URI from
# its own secret file (PGRST_DB_URI=@/run/secrets/postgrest_db_uri). Minting them
# anywhere else independently is what causes "password authentication failed for
# user authenticator". This is the ONLY place either is written.
set_postgres_password() {
    replace_secret postgres_password "$1"
    replace_secret postgrest_db_uri  "postgres://authenticator:${1}@postgres:5432/postgres"
}

# ── init ──────────────────────────────────────────────────────────────────────
cmd_init() {
    require_cmds docker openssl
    local force=0; [ "${1:-}" = "--force" ] && force=1
    
    swarm_active || { log "Initializing Docker Swarm"; docker swarm init >/dev/null; }
    
    if [ ! -f "$ENV_FILE" ]; then
        log "Creating .env from .env.example — edit it before deploying"
        cp "$ENV_EXAMPLE" "$ENV_FILE"
    fi
    
    # Bind-mounted scripts must be executable on the host.
    chmod +x "$HERE/bin/secrets-entrypoint.sh" "$HERE/postgres/"*.sh
    
    log "Creating Docker secrets (skipping any that already exist)"
    local s

    # Postgres password + the PostgREST URI derived from it — always minted together.
    # On --force, or if EITHER is missing, (re)mint the pair so they can't drift apart.
    if secret_exists postgres_password && secret_exists postgrest_db_uri && [ "$force" -ne 1 ]; then
        log "secret 'postgres_password' exists — skip (postgrest_db_uri tracks it)"
    else
        set_postgres_password "$(openssl rand -hex 24)"
    fi

    for s in $GENERATED_SECRETS; do
        if secret_exists "$s" && [ "$force" -ne 1 ]; then log "secret '$s' exists — skip"; continue; fi
        case "$s" in
            session_secret)      replace_secret "$s" "$(openssl rand -hex 32)" ;;
            # Inngest self-host rejects the signkey- prefix — plain hex only.
            inngest_signing_key) replace_secret "$s" "$(openssl rand -hex 32)" ;;
            inngest_event_key)   replace_secret "$s" "$(openssl rand -hex 16)" ;;
        esac
    done

    # Supabase key trio — must be minted together (anon/service signed by jwt_secret).
    if secret_exists jwt_secret && [ "$force" -ne 1 ]; then
        log "secret 'jwt_secret' exists — skipping Supabase trio"
    else
        log "Generating Supabase key trio"
        local trio jwt anon svc
        trio=$(bash "$HERE/scripts/gen-supabase-keys.sh")
        jwt=$(printf '%s\n' "$trio"  | sed -n 's/^SUPABASE_JWT_SECRET=//p')
        anon=$(printf '%s\n' "$trio" | sed -n 's/^SUPABASE_ANON_KEY=//p')
        svc=$(printf '%s\n' "$trio"  | sed -n 's/^SUPABASE_SERVICE_ROLE_KEY=//p')
        replace_secret jwt_secret "$jwt"
        replace_secret anon_key "$anon"
        replace_secret service_role_key "$svc"
    fi
    
    # Operator-supplied secrets — placeholders so the stack deploys; replace later.
    # resend_api_key is optional (marketing contacts / legacy fallback); app email
    # sends over SMTP using the shared GOTRUE_SMTP_* config + smtp_password.
    for s in $SUPPLIED_SECRETS; do
        secret_exists "$s" && continue
        case "$s" in
            # Seeded EMPTY on purpose: a truthy placeholder would engage the
            # legacy Resend SMTP fallback with junk credentials instead of
            # letting email no-op.
            resend_api_key) create_secret "$s" "" ;;
            smtp_password)  create_secret "$s" "change_me" ;;
            # Ignored while SAML_ENABLED=false; replace before enabling SAML.
            saml_private_key) create_secret "$s" "change_me" ;;
        esac
    done
    
    log "Secrets ready."
    warn "Next:"
    warn "  1. Edit $ENV_FILE — set CARBON_REPO, the *_HOST/*_URL, ACME_EMAIL, SMTP."
    warn "  2. Set real secrets:  $SCRIPT_NAME secret smtp_password xxx   (resend_api_key is optional)"
    warn "  3. $SCRIPT_NAME up"
}

# ── build ──────────────────────────────────────────────────────────────────────
cmd_build() {
    require_cmds docker; load_env
    if [ -n "${CARBON_REGISTRY:-}" ]; then
        log "CARBON_REGISTRY set — skipping local build (images pulled from registry on deploy)"
        return 0
    fi
    log "Building erp image ($CARBON_IMAGE_ERP) — needs ~8 GB RAM (NODE_OPTIONS)"
    docker build --build-arg APP=erp -t "$CARBON_IMAGE_ERP" "$CARBON_REPO"
    log "Building mes image ($CARBON_IMAGE_MES)"
    docker build --build-arg APP=mes -t "$CARBON_IMAGE_MES" "$CARBON_REPO"
    log "Building bootstrap image ($CARBON_IMAGE_BOOTSTRAP)"
    docker build --target bootstrap -t "$CARBON_IMAGE_BOOTSTRAP" "$CARBON_REPO"
}

# ── deploy ─────────────────────────────────────────────────────────────────────
cmd_deploy() {
    require_cmds docker; load_env
    swarm_active || error "swarm not active — run '$SCRIPT_NAME init'"
    # Every external secret must exist or `docker stack deploy` fails cryptically.
    local s missing=""
    for s in $ALL_SECRETS; do secret_exists "$s" || missing="$missing $s"; done
    [ -z "$missing" ] || error "missing secret(s):${missing} — run '$SCRIPT_NAME init'"
    log "Deploying stack '$STACK_NAME'"
    # --resolve-image never: use the locally built tags as-is for the no-registry path.
    local resolve=never; [ -n "${CARBON_REGISTRY:-}" ] && resolve=always
    docker stack deploy --detach=true --resolve-image "$resolve" -c "$STACK_FILE" "$STACK_NAME"
}

# Wait until a service has a healthy task. Matches on the exact Swarm service-name
# label, NOT a name substring — `name=carbon_postgres` also matches carbon_postgres-t
# and would report a false positive.
wait_healthy() {
    local svc="$1" timeout="${2:-180}" name="${STACK_NAME}_$1" waited=0
    log "Waiting for $name to be healthy (timeout ${timeout}s)"
    while [ "$waited" -lt "$timeout" ]; do
        if docker ps --filter "label=com.docker.swarm.service.name=$name" \
                      --filter health=healthy -q | grep -q .; then
            log "$name healthy"; return 0
        fi
        sleep 5; waited=$((waited + 5))
    done
    error "$name not healthy after ${timeout}s — check '$SCRIPT_NAME logs $svc'"
}

# ── migrate (ephemeral run-once job on the stack network) ───────────────────────
cmd_migrate() {
    require_cmds docker; load_env
    wait_healthy postgres 180
    wait_healthy storage 180
    local job="${STACK_NAME}_migrate"
    docker service rm "$job" >/dev/null 2>&1 || true
    log "Running migrations (supabase migration up)"
    docker service create \
    --name "$job" \
    --mode replicated-job \
    --detach=true \
    --network "${STACK_NAME}_internal" \
    --secret postgres_password \
    --restart-condition on-failure \
    --restart-max-attempts 10 \
    --env PGSSLMODE=disable \
    --workdir /repo/packages/database \
    "$CARBON_IMAGE_BOOTSTRAP" \
    sh -c 'pnpm exec supabase migration up --include-all \
    --db-url "postgresql://supabase_admin:$(cat /run/secrets/postgres_password)@postgres:5432/postgres"' >/dev/null

    # Wait for the one-shot job to finish, then ALWAYS surface its logs and abort on
    # failure — a detached job otherwise dies silently and `up` would roll on regardless.
    # Succeed on Complete; only fail once retries are exhausted (no task still active),
    # so a transient mid-retry Failed doesn't abort a job that would have succeeded.
    local waited=0 timeout=600 outcome="" states=""
    while [ "$waited" -lt "$timeout" ]; do
        states=$(docker service ps "$job" --format '{{.CurrentState}}' 2>/dev/null)
        if printf '%s\n' "$states" | grep -q '^Complete'; then outcome=complete; break; fi
        if ! printf '%s\n' "$states" | grep -qE '^(Running|Ready|Preparing|Assigned|Starting|New|Pending|Accepted)' \
           && printf '%s\n' "$states" | grep -qE '^(Failed|Rejected)'; then outcome=failed; break; fi
        sleep 5; waited=$((waited + 5))
    done

    if [ "$outcome" != complete ]; then
        warn "Migration job ${outcome:-timed out}. Logs:"
        docker service logs "$job" 2>&1 | tail -60 >&2 || true
        docker service rm "$job" >/dev/null 2>&1 || true
        error "Migrations failed — see logs above."
    fi
    docker service rm "$job" >/dev/null 2>&1 || true
    log "Migrations applied"
    set_inngest_event_url
}

# Postgres posts its events (event queue, embeddings, notifications) to Inngest
# at the Vault secret `inngest_event_url`; without it they are never delivered.
# The event key is a Swarm secret the host cannot read back, so it is read from
# the running erp task. Idempotent; re-run `migrate` after rotating the key.
set_inngest_event_url() {
    local erp pg key
    erp="$(docker ps -qf "name=${STACK_NAME}_erp.1" | head -1)"
    pg="$(docker ps -qf "name=${STACK_NAME}_postgres.1" | head -1)"
    if [ -z "$erp" ] || [ -z "$pg" ]; then
        warn "erp or postgres is not running — database events are not wired yet; re-run '$SCRIPT_NAME migrate' once the stack is up"
        return 0
    fi
    key="$(docker exec "$erp" cat /run/secrets/inngest_event_key)" || {
        warn "Could not read inngest_event_key from the erp task — database events will not be delivered"
        return 0
    }
    printf "SELECT public.set_inngest_event_url('http://inngest:8288/e/%s');\n" "$key" \
        | docker exec -i "$pg" psql -q -U postgres -d postgres -v ON_ERROR_STOP=1 >/dev/null \
        && log "Database events wired to Inngest" \
        || warn "Could not set the Inngest event URL — database events will not be delivered"
}

# ── up (build + deploy + migrate + roll apps) ───────────────────────────────────
cmd_up() {
    cmd_build
    cmd_deploy
    cmd_migrate
    load_env
    log "Rolling apps now that the schema exists"
    docker service update --force --detach=true "${STACK_NAME}_erp" >/dev/null
    docker service update --force --detach=true "${STACK_NAME}_mes" >/dev/null
    log "Stack up. Check: $SCRIPT_NAME status"
}

# ── secret (create or rotate one secret) ────────────────────────────────────────
# Value comes from the argument, or from stdin when piped:
#   deploy.sh secret resend_api_key re_xxx
#   printf '%s' "$VALUE" | deploy.sh secret resend_api_key
cmd_secret() {
    require_cmds docker
    local name="${1:?usage: $SCRIPT_NAME secret <name> <value>}"
    local val
    if [ "$#" -ge 2 ]; then val="$2"
    elif [ ! -t 0 ]; then val=$(cat)
    else error "usage: $SCRIPT_NAME secret $name <value>  (or pipe the value on stdin)"
    fi
    [ -n "$val" ] || error "empty value for secret '$name'"
    # Rotating the DB password must also re-mint the derived PostgREST URI, or
    # postgrest fails auth. Note: the DB only adopts a new password on a fresh
    # pgdata volume, so a live rotation needs `down --volumes` + `init --force`.
    if [ "$name" = "postgres_password" ]; then
        set_postgres_password "$val"
        return 0
    fi
    replace_secret "$name" "$val"
}

cmd_status() {
    load_env
    docker stack services "$STACK_NAME"
    echo
    docker stack ps "$STACK_NAME" --no-trunc --filter desired-state=running
}

cmd_logs() {
    load_env
    local svc="${1:?usage: $SCRIPT_NAME logs <service>}"
    docker service logs -f --tail 200 "${STACK_NAME}_${svc}"
}

cmd_down() {
    require_cmds docker; load_env
    log "Removing stack '$STACK_NAME'"
    docker stack rm "$STACK_NAME"
    if [ "${1:-}" = "--volumes" ]; then
        warn "Waiting for tasks to stop, then deleting volumes (DATA LOSS)"
        local waited=0
        while docker ps -q --filter "label=com.docker.stack.namespace=${STACK_NAME}" | grep -q .; do
            [ "$waited" -ge 60 ] && break
            sleep 3; waited=$((waited + 3))
        done
        docker volume ls -q --filter "name=${STACK_NAME}_" | xargs -r docker volume rm >/dev/null 2>&1 \
            || warn "some volumes still in use — re-run '$SCRIPT_NAME down --volumes'"
        log "Volumes removed"
    fi
}

# Print the "Usage:" comment block (from that line until the first non-comment line).
usage() { awk '/^# Usage:/{f=1} f&&!/^#/{exit} f{sub(/^# ?/,"");print}' "$0"; exit "${1:-0}"; }

main() {
    local sub="${1:-}"; [ $# -gt 0 ] && shift || true
    case "$sub" in
        init)    cmd_init "$@" ;;
        build)   cmd_build "$@" ;;
        deploy)  cmd_deploy "$@" ;;
        up)      cmd_up "$@" ;;
        migrate) cmd_migrate "$@" ;;
        secret)  cmd_secret "$@" ;;
        status)  cmd_status "$@" ;;
        logs)    cmd_logs "$@" ;;
        down)    cmd_down "$@" ;;
        -h|--help|help|"") usage 0 ;;
        *) error "unknown command: $sub (try '$SCRIPT_NAME --help')" ;;
    esac
}

main "$@"