Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(workspace): stabilise the development stack (#2024)

Hampus5d2e5932a492
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index f69a4b706..e3c8a9aea 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -7,7 +7,6 @@ ARG USER_UID=1000
ARG USER_GID=1000
ARG NODE_MAJOR=24
ARG ELP_VERSION=2026-02-27
-ARG HELM_VERSION=4.2.0
ARG PNPM_VERSION=10.29.3
ARG WASM_BINDGEN_VERSION=0.2.122
@@ -15,8 +14,8 @@ ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
+ acl \
bash \
- brotli \
build-essential \
ca-certificates \
clang \
@@ -42,13 +41,13 @@ RUN apt-get update \
libfido2-dev \
libgbm1 \
libgtk-3-0 \
+ libimage-exiftool-perl \
libnotify4 \
libnss3 \
libpipewire-0.3-dev \
libpulse-dev \
libsecret-1-0 \
libudev-dev \
- libuv1-dev \
libcurl4-openssl-dev \
libswresample-dev \
libswscale-dev \
@@ -71,14 +70,12 @@ RUN apt-get update \
ninja-build \
openssh-client \
pkg-config \
- protobuf-compiler \
python3 \
python3-pip \
- rsync \
+ python3-venv \
sudo \
rpm \
unzip \
- webp \
xz-utils \
xdg-utils \
zstd \
@@ -90,6 +87,7 @@ RUN apt-get update \
bat \
btop \
docker-cli \
+ docker-compose \
dnsutils \
fd-find \
gdb \
@@ -122,24 +120,12 @@ RUN apt-get update \
&& ln -sf /usr/bin/batcat /usr/local/bin/bat \
&& rm -rf /var/lib/apt/lists/*
-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \
+RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \
&& apt-get install -y --no-install-recommends nodejs \
&& rm -rf /var/lib/apt/lists/* \
&& corepack enable
-RUN ARCH="$(dpkg --print-architecture)" \
- && case "$ARCH" in \
- amd64) HELM_ARCH="amd64" ;; \
- arm64) HELM_ARCH="arm64" ;; \
- *) echo "Unsupported architecture for Helm: $ARCH" >&2; exit 1 ;; \
- esac \
- && curl -fsSL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${HELM_ARCH}.tar.gz" -o /tmp/helm.tgz \
- && tar -C /tmp -xzf /tmp/helm.tgz "linux-${HELM_ARCH}/helm" \
- && mv "/tmp/linux-${HELM_ARCH}/helm" /usr/local/bin/helm \
- && chmod +x /usr/local/bin/helm \
- && rm -rf /tmp/helm.tgz "/tmp/linux-${HELM_ARCH}"
-
-RUN python3 -m pip install --break-system-packages --no-cache-dir awscli cqlsh
+RUN python3 -m pip install --break-system-packages --no-cache-dir awscli
COPY tools/fonts/requirements.txt /tmp/fluxer-fonts-requirements.txt
RUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer-fonts-requirements.txt \
@@ -147,18 +133,13 @@ RUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer
&& pyftsubset --help >/dev/null \
&& python3 -c "import fontTools, brotli"
-RUN if ! command -v rebar3 >/dev/null 2>&1; then \
- curl -fsSL https://s3.amazonaws.com/rebar3/rebar3 -o /usr/local/bin/rebar3 \
- && chmod +x /usr/local/bin/rebar3; \
- fi
-
RUN ARCH="$(dpkg --print-architecture)" \
&& case "$ARCH" in \
amd64) ELP_ARCH="x86_64" ;; \
arm64) ELP_ARCH="aarch64" ;; \
*) echo "Unsupported architecture for ELP: $ARCH" >&2; exit 1 ;; \
esac \
- && curl -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \
+ && curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \
&& tar -C /usr/local/bin -xzf /tmp/elp.tgz elp \
&& chmod +x /usr/local/bin/elp \
&& rm /tmp/elp.tgz
@@ -179,16 +160,13 @@ ENV DOCKER_HOST="unix:///var/run/docker.sock" \
ENV CC_wasm32_unknown_unknown="clang" \
AR_wasm32_unknown_unknown="llvm-ar"
-RUN curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile default --component clippy,rustfmt \
+RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --component clippy,rustfmt \
&& rustup target add wasm32-unknown-unknown \
&& cargo install cargo-watch --locked \
- && cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked
+ && cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked \
+ && rm -rf "/home/${USERNAME}/.cargo/registry" "/home/${USERNAME}/.cargo/git"
RUN corepack prepare "pnpm@${PNPM_VERSION}" --activate \
&& pnpm --version
-RUN sudo apt-get update \
- && sudo apt-get install -y --no-install-recommends python3-venv \
- && sudo rm -rf /var/lib/apt/lists/*
-
WORKDIR /workspaces/fluxer
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 16a810fb4..42b866038 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -5,20 +5,17 @@
"workspaceFolder": "/workspaces/fluxer",
"shutdownAction": "stopCompose",
"remoteUser": "vscode",
+ "hostRequirements": {
+ "cpus": 4,
+ "memory": "8gb",
+ "storage": "32gb"
+ },
"remoteEnv": {
"DOCKER_HOST": "unix:///var/run/docker.sock"
},
"runServices": ["workspace", "postgres", "valkey", "nats", "livekit", "meilisearch", "mailpit"],
- "forwardPorts": [
- 3000, 8088, 8080, 8771, 8082, 3010, 3020, 8100, 8101, 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111,
- 8112, 8113, 8114, 8115, 8116, 8117, 8118, 8119, 8120, 8121, 8122, 8123, 8124, 8125, 3900, 8888, 9333, 9340, 23646,
- 4222, 7700, 7880, 7900, 9200, 8000
- ],
+ "forwardPorts": [3000, 8088, 8080, 8771, 8082, 8773, 3010, 3020, 8333],
"portsAttributes": {
- "8000": {
- "label": "Zensical docs",
- "onAutoForward": "openBrowserOnce"
- },
"8088": {
"label": "Fluxer dev proxy",
"onAutoForward": "notify"
@@ -29,36 +26,15 @@
"3020": {
"label": "Fluxer admin"
},
- "8100": {
- "label": "Fluxer Rust service health"
- },
- "3900": {
+ "8333": {
"label": "SeaweedFS S3"
},
- "8888": {
- "label": "SeaweedFS filer"
- },
- "9333": {
- "label": "SeaweedFS master"
- },
- "9340": {
- "label": "SeaweedFS volume"
- },
- "23646": {
- "label": "SeaweedFS admin"
- },
- "7880": {
- "label": "LiveKit"
- },
- "7700": {
- "label": "Meilisearch"
- },
- "9200": {
- "label": "Elasticsearch"
+ "8773": {
+ "label": "Fluxer app proxy"
}
},
- "postCreateCommand": "sudo chown -R vscode:vscode /workspaces/fluxer/target && find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -exec sudo chown -R vscode:vscode {} + && sudo chown -R vscode:vscode /home/vscode/.local/share/pnpm && cargo run -p fluxer-dev -- bootstrap",
- "postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && cargo run -p fluxer-dev -- post-start && bash /workspaces/fluxer/fluxer_docs/serve.sh --daemon",
+ "postCreateCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- bootstrap",
+ "postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- post-start",
"customizations": {
"vscode": {
"settings": {
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
index d6ec6eb44..aec056885 100644
--- a/.devcontainer/docker-compose.yml
+++ b/.devcontainer/docker-compose.yml
@@ -1,5 +1,3 @@
-name: fluxer-dev
-
services:
workspace:
build:
@@ -7,15 +5,29 @@ services:
dockerfile: .devcontainer/Dockerfile
command: sleep infinity
init: true
- env_file:
- - ../config/env/development.env
environment:
- FLUXER_SEARCH_ENGINE: meilisearch
- FLUXER_SEARCH_URL: http://meilisearch:7700
- FLUXER_SEARCH_API_KEY: fluxer-dev-meilisearch
- FLUXER_POSTGRES_HOST: postgres
- FLUXER_SELF_HOSTED: "true"
DOCKER_HOST: unix:///var/run/docker.sock
+ npm_config_store_dir: /home/vscode/.local/share/pnpm/store
+ FLUXER_PUBLIC_PORT: "${FLUXER_DEV_PROXY_PORT:-8088}"
+ FLUXER_PUBLIC_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
+ FLUXER_API_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"
+ FLUXER_API_CLIENT_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"
+ FLUXER_APP_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
+ FLUXER_GATEWAY_ENDPOINT: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/gateway"
+ FLUXER_MEDIA_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
+ FLUXER_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
+ FLUXER_ADMIN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin"
+ FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
+ FLUXER_S3_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
+ FLUXER_LIVEKIT_URL: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/livekit"
+ FLUXER_LIVEKIT_INTERNAL_URL: "http://livekit:7880"
+ FLUXER_LIVEKIT_WEBHOOK_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api/webhooks/livekit"
+ FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
+ FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
+ FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
+ FLUXER_ADMIN_OAUTH_REDIRECT_URI: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin/oauth2_callback"
+ FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: "http://localhost,http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
+ PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"
volumes:
- ..:/workspaces/fluxer:cached
- type: volume
@@ -229,6 +241,7 @@ services:
volume:
nocopy: true
- pnpm-store:/home/vscode/.local/share/pnpm/store
+ - docs-venv:/workspaces/fluxer/fluxer_docs/.venv
- cargo-registry:/home/vscode/.cargo/registry
- cargo-git:/home/vscode/.cargo/git
- rust-target:/workspaces/fluxer/target
@@ -236,45 +249,31 @@ services:
source: ${FLUXER_DOCKER_SOCKET:-/var/run/docker.sock}
target: /var/run/docker.sock
ports:
- - "${FLUXER_DEV_DOCS_PORT:-8000}:8000"
- - "${FLUXER_DEV_RSPACK_PORT:-3000}:3000"
- - "${FLUXER_DEV_PROXY_PORT:-8088}:8088"
- - "${FLUXER_DEV_APP_PROXY_PORT:-8080}:8080"
- - "${FLUXER_DEV_API_PORT:-8771}:8771"
- - "${FLUXER_DEV_GATEWAY_PORT:-8082}:8082"
- - "${FLUXER_DEV_MARKETING_PORT:-3010}:3010"
- - "${FLUXER_DEV_ADMIN_PORT:-3020}:3020"
- - "${FLUXER_DEV_RUST_SERVICE_PORTS:-8100-8125}:8100-8125"
- - "${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333"
- - "${FLUXER_DEV_SEAWEEDFS_FILER_PORT:-8888}:8888"
- - "${FLUXER_DEV_SEAWEEDFS_MASTER_PORT:-9333}:9333"
- - "${FLUXER_DEV_SEAWEEDFS_VOLUME_PORT:-9340}:9340"
- - "${FLUXER_DEV_SEAWEEDFS_ADMIN_PORT:-23646}:23646"
+ - "127.0.0.1:${FLUXER_DEV_RSPACK_PORT:-3000}:3000"
+ - "127.0.0.1:${FLUXER_DEV_PROXY_PORT:-8088}:8088"
+ - "127.0.0.1:${FLUXER_DEV_API_PORT:-8080}:8080"
+ - "127.0.0.1:${FLUXER_DEV_GATEWAY_PORT:-8771}:8771"
+ - "127.0.0.1:${FLUXER_DEV_MEDIA_PROXY_PORT:-8082}:8082"
+ - "127.0.0.1:${FLUXER_DEV_APP_PROXY_PORT:-8773}:8773"
+ - "127.0.0.1:${FLUXER_DEV_MARKETING_PORT:-3010}:3010"
+ - "127.0.0.1:${FLUXER_DEV_ADMIN_PORT:-3020}:3020"
+ - "127.0.0.1:${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333"
depends_on:
- - postgres
- - valkey
- - nats
- - livekit
- - meilisearch
- - mailpit
+ postgres:
+ condition: service_healthy
+ valkey:
+ condition: service_started
+ nats:
+ condition: service_started
+ livekit:
+ condition: service_started
+ meilisearch:
+ condition: service_healthy
+ mailpit:
+ condition: service_started
extra_hosts:
- "host.docker.internal:host-gateway"
- cassandra:
- image: cassandra:5.0.8
- profiles:
- - full
- environment:
- CASSANDRA_CLUSTER_NAME: fluxer-dev
- CASSANDRA_DC: datacenter1
- CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch
- HEAP_NEWSIZE: 128M
- MAX_HEAP_SIZE: 768M
- volumes:
- - cassandra-data:/var/lib/cassandra
- ports:
- - "${FLUXER_DEV_CASSANDRA_PORT:-9042}:9042"
-
postgres:
image: postgres:16-alpine
environment:
@@ -284,13 +283,19 @@ services:
volumes:
- postgres-data:/var/lib/postgresql/data
ports:
- - "${FLUXER_DEV_POSTGRES_PORT:-5432}:5432"
+ - "127.0.0.1:${FLUXER_DEV_POSTGRES_PORT:-5432}:5432"
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U fluxer -d fluxer"]
+ interval: 2s
+ timeout: 5s
+ retries: 30
+ start_period: 5s
valkey:
image: valkey/valkey:8.1.7-alpine
command: ["valkey-server", "--save", "", "--appendonly", "no"]
ports:
- - "${FLUXER_DEV_VALKEY_PORT:-6379}:6379"
+ - "127.0.0.1:${FLUXER_DEV_VALKEY_PORT:-6379}:6379"
nats:
image: nats:2.14.2-alpine
@@ -298,33 +303,22 @@ services:
volumes:
- nats-data:/data
ports:
- - "${FLUXER_DEV_NATS_PORT:-4222}:4222"
- - "${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222"
+ - "127.0.0.1:${FLUXER_DEV_NATS_PORT:-4222}:4222"
+ - "127.0.0.1:${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222"
livekit:
image: livekit/livekit-server:v1.12.0
command: ["--config", "/etc/livekit.yaml", "--bind", "0.0.0.0"]
- volumes:
- - ./livekit.yaml:/etc/livekit.yaml:ro
- ports:
- - "${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880"
- - "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:7881"
- - "${FLUXER_DEV_LIVEKIT_UDP_PORTS:-7882-7892}:7882-7892/udp"
-
- elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:9.3.2
- profiles:
- - full
environment:
- discovery.type: single-node
- xpack.security.enabled: "true"
- xpack.security.http.ssl.enabled: "false"
- ELASTIC_PASSWORD: fluxer-dev-elasticsearch
- ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+ LIVEKIT_RTC_TCP_PORT: "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}"
+ LIVEKIT_RTC_UDP_PORT_START: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}"
+ LIVEKIT_RTC_UDP_PORT_END: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}"
volumes:
- - elasticsearch-data:/usr/share/elasticsearch/data
+ - ./livekit.yaml:/etc/livekit.yaml:ro
ports:
- - "${FLUXER_DEV_ELASTICSEARCH_PORT:-9200}:9200"
+ - "127.0.0.1:${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880"
+ - "127.0.0.1:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}"
+ - "127.0.0.1:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}/udp"
meilisearch:
image: getmeili/meilisearch:v1.12
@@ -334,7 +328,13 @@ services:
volumes:
- meilisearch-data:/meili_data
ports:
- - "${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700"
+ - "127.0.0.1:${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700"
+ healthcheck:
+ test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:7700/health"]
+ interval: 2s
+ timeout: 5s
+ retries: 30
+ start_period: 5s
mailpit:
image: axllent/mailpit:v1.30
@@ -349,6 +349,7 @@ services:
volumes:
pnpm-store:
+ docs-venv:
root-node-modules:
fluxer-api-node-modules:
fluxer-app-node-modules:
@@ -394,9 +395,7 @@ volumes:
cargo-registry:
cargo-git:
rust-target:
- cassandra-data:
nats-data:
- elasticsearch-data:
meilisearch-data:
mailpit-data:
postgres-data:
diff --git a/.devcontainer/fix-docker-socket.sh b/.devcontainer/fix-docker-socket.sh
index e0c3fdf59..ea864717f 100755
--- a/.devcontainer/fix-docker-socket.sh
+++ b/.devcontainer/fix-docker-socket.sh
@@ -1,10 +1,10 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
-set -uo pipefail
+set -euo pipefail
SOCKET="${DOCKER_SOCKET:-/var/run/docker.sock}"
-USER_NAME="${USER:-vscode}"
+USER_NAME="$(id -un)"
if [ ! -S "$SOCKET" ]; then
echo "fix-docker-socket: no socket at $SOCKET; skipping (Docker-in-devcontainer will not work)"
@@ -16,31 +16,11 @@ if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then
exit 0
fi
-socket_gid="$(stat -c '%g' "$SOCKET" 2>/dev/null || echo "")"
-if [ -z "$socket_gid" ]; then
- echo "fix-docker-socket: could not stat $SOCKET; skipping" >&2
- exit 0
-fi
-
-sudo sh -c '
- set -e
- gid="$1"
- user="$2"
- socket="$3"
- if ! getent group "$gid" >/dev/null 2>&1; then
- groupadd --gid "$gid" docker-host
- fi
- group_name="$(getent group "$gid" | cut -d: -f1)"
- usermod --append --groups "$group_name" "$user"
- chgrp "$gid" "$socket"
- chmod g+rw "$socket"
-' sh "$socket_gid" "$USER_NAME" "$SOCKET" || {
- echo "fix-docker-socket: could not adjust $SOCKET; run docker with sudo" >&2
- exit 0
-}
+sudo setfacl --modify "user:${USER_NAME}:rw" "$SOCKET"
if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then
- echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME (gid $socket_gid)"
+ echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME"
else
- echo "fix-docker-socket: $SOCKET still unreachable as $USER_NAME; run docker with sudo" >&2
+ echo "fix-docker-socket: $SOCKET is still unreachable as $USER_NAME" >&2
+ exit 1
fi
diff --git a/.devcontainer/fix-workspace-permissions.sh b/.devcontainer/fix-workspace-permissions.sh
new file mode 100644
index 000000000..0ef0c5d4e
--- /dev/null
+++ b/.devcontainer/fix-workspace-permissions.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+set -euo pipefail
+
+owner="$(id -u):$(id -g)"
+
+repair_tree() {
+ local path="$1"
+ local unwritable
+ if [ ! -d "$path" ]; then
+ echo "fix-workspace-permissions: expected mount is missing: $path" >&2
+ exit 1
+ fi
+ unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)"
+ if [ ! -w "$path" ] || [ -n "$unwritable" ]; then
+ sudo find "$path" -xdev \( -type d -o -type f \) -exec chown "$owner" {} +
+ fi
+ unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)"
+ if [ ! -w "$path" ] || [ -n "$unwritable" ]; then
+ echo "fix-workspace-permissions: $path is not writable as $(id -un)" >&2
+ exit 1
+ fi
+}
+
+for path in \
+ /workspaces/fluxer/target \
+ /home/vscode/.cargo/registry \
+ /home/vscode/.cargo/git \
+ /home/vscode/.local \
+ /home/vscode/.local/share/pnpm/store \
+ /workspaces/fluxer/fluxer_docs/.venv; do
+ repair_tree "$path"
+done
+
+while IFS= read -r -d '' path; do
+ if mountpoint -q "$path"; then
+ repair_tree "$path"
+ fi
+done < <(find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -print0)
diff --git a/.devcontainer/livekit.yaml b/.devcontainer/livekit.yaml
index 552df10d2..f49d40c42 100644
--- a/.devcontainer/livekit.yaml
+++ b/.devcontainer/livekit.yaml
@@ -1,11 +1,10 @@
port: 7880
keys:
- devkey: secret
+ devkey: fluxer-livekit-development-secret
rtc:
tcp_port: 7881
- udp_port: 7882-7892
node_ip: 127.0.0.1
use_mdns: true
stun_servers:
diff --git a/config/env/development.env b/config/env/development.env
index 5cf6d0547..7a4867130 100644
--- a/config/env/development.env
+++ b/config/env/development.env
@@ -30,12 +30,6 @@ FLUXER_POSTGRES_PASSWORD=fluxer
FLUXER_POSTGRES_SSL=false
FLUXER_POSTGRES_MAX_CONNECTIONS=20
FLUXER_POSTGRES_KV_TABLE=fluxer_kv
-FLUXER_CASSANDRA_HOSTS=cassandra
-FLUXER_CASSANDRA_PORT=9042
-FLUXER_CASSANDRA_KEYSPACE=fluxer
-FLUXER_CASSANDRA_LOCAL_DC=datacenter1
-FLUXER_CASSANDRA_USERNAME=fluxer
-FLUXER_CASSANDRA_PASSWORD=fluxer
FLUXER_KV_URL=redis://valkey:6379/0
FLUXER_NATS_URL=nats://nats:4222
FLUXER_NATS_JETSTREAM_URL=nats://nats:4222
@@ -66,8 +60,9 @@ FLUXER_S3_BUCKET_STATIC=fluxer-static
FLUXER_LIVEKIT_ENABLED=true
FLUXER_LIVEKIT_URL=ws://localhost:8088/livekit
+FLUXER_LIVEKIT_INTERNAL_URL=http://localhost:7880
FLUXER_LIVEKIT_API_KEY=devkey
-FLUXER_LIVEKIT_API_SECRET=secret
+FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secret
FLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekit
FLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686}
diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml
index d3484fe61..57fc260ad 100644
--- a/deploy/self-hosting/docker-compose.yml
+++ b/deploy/self-hosting/docker-compose.yml
@@ -47,6 +47,7 @@ x-fluxer-env: &fluxer-env
FLUXER_LIVEKIT_ENABLED: "true"
FLUXER_LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}
FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}
+ FLUXER_LIVEKIT_INTERNAL_URL: http://livekit:7880
FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit
FLUXER_LIVEKIT_DEFAULT_REGION: '{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}'
diff --git a/fluxer_api/src/api/App.ts b/fluxer_api/src/api/App.ts
index a35882ba5..9d982b505 100644
--- a/fluxer_api/src/api/App.ts
+++ b/fluxer_api/src/api/App.ts
@@ -39,7 +39,7 @@ function AbuseAwareAppErrorHandler(err: Error, ctx: Context<HonoEnv>): Response
export async function createAPIApp(options: CreateAPIAppOptions): Promise<APIAppResult> {
const {config, logger} = options;
- const shutdownApiLifecycle = createShutdown(logger);
+ const shutdownApiLifecycle = createShutdown(config, logger);
setIsDevelopment(config.nodeEnv === 'development');
const routes = new Hono<HonoEnv>({strict: true});
configureMiddleware(routes, {
diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts
index e32b81674..422a4158e 100644
--- a/fluxer_api/src/api/Config.ts
+++ b/fluxer_api/src/api/Config.ts
@@ -306,6 +306,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
apiSecret: master.integrations.voice.api_secret,
webhookUrl: master.integrations.voice.webhook_url,
url: master.integrations.voice.url,
+ internalUrl: master.integrations.voice.internal_url,
defaultRegion: master.integrations.voice.default_region,
},
stripe: {
diff --git a/fluxer_api/src/api/app/APILifecycle.ts b/fluxer_api/src/api/app/APILifecycle.ts
index 2cfd66fee..9097416d0 100644
--- a/fluxer_api/src/api/app/APILifecycle.ts
+++ b/fluxer_api/src/api/app/APILifecycle.ts
@@ -36,6 +36,11 @@ import {JetStreamWorkerQueue} from '../worker/JetStreamWorkerQueue';
import {WorkerService} from '../worker/WorkerService';
let jsConnectionManager: JetStreamConnectionManager | null = null;
+
+function unsupportedDatabaseBackend(backend: never): never {
+ throw new Error(`Unsupported database backend during shutdown: ${String(backend)}`);
+}
+
export function createInitializer(config: APIConfig, logger: ILogger): () => Promise<void> {
return async (): Promise<void> => {
try {
@@ -131,7 +136,7 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro
try {
const kvDeletionQueue = getKVAccountDeletionQueue();
if (await kvDeletionQueue.needsRebuild()) {
- logger.warn('KV deletion queue needs rebuild, rebuilding...');
+ logger.info('KV deletion queue needs rebuild, rebuilding...');
await kvDeletionQueue.rebuildState();
} else {
logger.info('KV deletion queue state is healthy');
@@ -204,13 +209,13 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro
logger.info('API service initialization complete');
} catch (error) {
logger.error({error}, 'API service initialization failed');
- await createShutdown(logger)();
+ await createShutdown(config, logger)();
throw error;
}
};
}
-export function createShutdown(logger: ILogger): () => Promise<void> {
+export function createShutdown(config: APIConfig, logger: ILogger): () => Promise<void> {
return async (): Promise<void> => {
logger.info('Shutting down API service...');
if (jsConnectionManager) {
@@ -258,18 +263,26 @@ export function createShutdown(logger: ILogger): () => Promise<void> {
} catch (error) {
logger.error({error}, 'Error shutting down report service');
}
- try {
- setDatabaseQueryExecutor(null);
- await shutdownPostgres();
- logger.info('Postgres client shut down');
- } catch (error) {
- logger.error({error}, 'Error shutting down Postgres client');
- }
- try {
- await shutdownCassandra();
- logger.info('Cassandra client shut down');
- } catch (error) {
- logger.error({error}, 'Error shutting down Cassandra client');
+ setDatabaseQueryExecutor(null);
+ switch (config.database.backend) {
+ case 'postgres':
+ try {
+ await shutdownPostgres();
+ logger.info('Postgres client shut down');
+ } catch (error) {
+ logger.error({error}, 'Error shutting down Postgres client');
+ }
+ break;
+ case 'cassandra':
+ try {
+ await shutdownCassandra();
+ logger.info('Cassandra client shut down');
+ } catch (error) {
+ logger.error({error}, 'Error shutting down Cassandra client');
+ }
+ break;
+ default:
+ unsupportedDatabaseBackend(config.database.backend);
}
logger.info('API service shutdown complete');
};
diff --git a/fluxer_api/src/api/app/ControllerRegistry.ts b/fluxer_api/src/api/app/ControllerRegistry.ts
index 4db31caa0..28c851d9b 100644
--- a/fluxer_api/src/api/app/ControllerRegistry.ts
+++ b/fluxer_api/src/api/app/ControllerRegistry.ts
@@ -66,7 +66,9 @@ export function registerControllers(routes: HonoApp, config: APIConfig): void {
TestHarnessController(routes);
}
UserController(routes);
- registerInboundSmsWebhook(routes);
+ if (config.sms.enabled) {
+ registerInboundSmsWebhook(routes);
+ }
WebhookController(routes);
OAuth2Controller(routes);
OAuth2ApplicationsController(routes);
diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts
index 3b685ce92..58ea5fd50 100644
--- a/fluxer_api/src/api/config/APIConfig.ts
+++ b/fluxer_api/src/api/config/APIConfig.ts
@@ -204,6 +204,7 @@ export interface APIConfig {
apiSecret?: string;
webhookUrl?: string;
url?: string;
+ internalUrl?: string;
defaultRegion?: {
id: string;
name: string;
diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
index db05aea5e..8ae05ca42 100644
--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
@@ -19,6 +19,8 @@ interface PageState {
const VALUE_SEPARATOR = '\u001f';
const ENCODED_TYPE_KEY = '__fluxer_type';
+const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;
+const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';
function normalizeCql(cql: string): string {
return cql.replace(/\s+/g, ' ').trim();
@@ -354,8 +356,13 @@ function parseEqWhere(whereSql: string, cql: string): ReadonlyArray<EqWhereExpr>
}
export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<void> {
- const table = quoteIdentifier(client.kvTable());
- await client.query(`
+ const kvTable = client.kvTable();
+ const table = quoteIdentifier(kvTable);
+ await client.transaction(async (db) => {
+ await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_LOCK_TIMEOUT]);
+ await db.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, kvTable]);
+ await db.query("SELECT set_config('statement_timeout', '0', true)");
+ await db.query(`
CREATE TABLE IF NOT EXISTS ${table} (
table_name text NOT NULL,
partition_key text NOT NULL,
@@ -365,28 +372,29 @@ CREATE TABLE IF NOT EXISTS ${table} (
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (table_name, row_key)
)`);
- await client.query(
- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,
- );
- await client.query(
- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,
- );
- await client.query(
- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,
- );
- await client.query(
- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,
- );
- await client.query(
- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
- );
- await client.query(`
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
+ );
+ await db.query(`
UPDATE ${table}
SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)
WHERE table_name = 'messages'
AND partition_key = row_key
AND split_part(row_key, chr(31), 3) <> ''`);
- await client.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_idx`)}`);
+ await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);
+ });
}
export async function pruneExpiredPostgresKvRows(client: IPostgresClient, batchSize = 5000): Promise<number> {
diff --git a/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts b/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts
index ec0174ad9..661175056 100644
--- a/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts
+++ b/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts
@@ -60,7 +60,7 @@ export class KVAccountDeletionQueueService {
}
async rebuildState(): Promise<void> {
- Logger.info('Starting deletion queue rebuild from Cassandra');
+ Logger.info('Starting deletion queue rebuild from primary database');
try {
await this.kvClient.del(QUEUE_KEY);
await this.kvClient.del(STATE_VERSION_KEY);
diff --git a/fluxer_api/src/api/infrastructure/LiveKitService.ts b/fluxer_api/src/api/infrastructure/LiveKitService.ts
index 8723c1443..597915364 100644
--- a/fluxer_api/src/api/infrastructure/LiveKitService.ts
+++ b/fluxer_api/src/api/infrastructure/LiveKitService.ts
@@ -109,6 +109,18 @@ function createRoomServiceClient(endpoint: string, apiKey: string, apiSecret: st
return client;
}
+function resolveRoomServiceEndpoint(server: VoiceServerRecord): string {
+ const defaultRegion = Config.voice.defaultRegion;
+ const internalUrl = Config.voice.internalUrl;
+ if (!defaultRegion || !internalUrl) {
+ return server.endpoint;
+ }
+ if (server.regionId !== defaultRegion.id || server.serverId !== `${defaultRegion.id}-server-1`) {
+ return server.endpoint;
+ }
+ return internalUrl;
+}
+
export class LiveKitService extends ILiveKitService {
private serverClients: Map<string, Map<string, ServerClientConfig>> = new Map();
private topology: VoiceTopology;
@@ -453,12 +465,13 @@ export class LiveKitService extends ILiveKitService {
const servers = this.topology.getServersForRegion(region.id);
const serverMap: Map<string, ServerClientConfig> = new Map();
for (const server of servers) {
+ const roomServiceEndpoint = resolveRoomServiceEndpoint(server);
serverMap.set(server.serverId, {
endpoint: server.endpoint,
apiKey: server.apiKey,
apiSecret: server.apiSecret,
isActive: server.isActive,
- roomServiceClient: createRoomServiceClient(server.endpoint, server.apiKey, server.apiSecret),
+ roomServiceClient: createRoomServiceClient(roomServiceEndpoint, server.apiKey, server.apiSecret),
});
}
newMap.set(region.id, serverMap);
diff --git a/fluxer_api/src/api/middleware/ServiceMiddleware.ts b/fluxer_api/src/api/middleware/ServiceMiddleware.ts
index 5fb8945aa..ca44b7db2 100644
--- a/fluxer_api/src/api/middleware/ServiceMiddleware.ts
+++ b/fluxer_api/src/api/middleware/ServiceMiddleware.ts
@@ -284,7 +284,7 @@ export function setInjectedAccountPolicyEvaluator(evaluator: IAccountPolicyEvalu
function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
if (_registrationRiskEvaluator) return _registrationRiskEvaluator;
if (!Config.risk.enabled) {
- Logger.warn(
+ Logger.info(
{},
'[ServiceMiddleware] integrations.risk_integration.enabled is false — account risk scoring is disabled',
);
diff --git a/fluxer_app_proxy/src/routes/assets_proxy.rs b/fluxer_app_proxy/src/routes/assets_proxy.rs
index 7d516d388..c2ed95abc 100644
--- a/fluxer_app_proxy/src/routes/assets_proxy.rs
+++ b/fluxer_app_proxy/src/routes/assets_proxy.rs
@@ -148,7 +148,7 @@ pub async fn proxy_assets(
response
}
-async fn serve_local_asset(
+pub(super) async fn serve_local_asset(
static_dir: &str,
relative_path: &str,
request_headers: &HeaderMap,
diff --git a/fluxer_app_proxy/src/routes/spa_index.rs b/fluxer_app_proxy/src/routes/spa_index.rs
index 487765422..96cfce35d 100644
--- a/fluxer_app_proxy/src/routes/spa_index.rs
+++ b/fluxer_app_proxy/src/routes/spa_index.rs
@@ -19,6 +19,7 @@ use axum::{
use std::path::Path;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use super::assets_proxy::serve_local_asset;
use super::spa_static::{CORS_ALLOW_ANY_VALUE, guess_mime, is_font_mime};
const ACCEPT_CH_VALUE: &str = "DPR, Sec-CH-DPR, Sec-CH-Width, Save-Data, ECT, Downlink";
@@ -35,6 +36,14 @@ pub async fn spa_catch_all(
if let Some(cache_control) = static_root_file_cache_control(request_path) {
return serve_static_file(&state.config.static_dir, request_path, cache_control).await;
}
+ if is_static_asset_path(request_path) {
+ return serve_local_asset(
+ &state.config.static_dir,
+ request_path.trim_start_matches('/'),
+ &headers,
+ )
+ .await;
+ }
serve_spa_index(&state, &headers, request_path).await
}
@@ -42,6 +51,16 @@ pub async fn spa_catch_all(
const CRAWL_CONTROL_CACHE_CONTROL: &str = "public, max-age=300, must-revalidate";
const STATIC_ROOT_FILES: &[(&str, &str)] = &[("/robots.txt", CRAWL_CONTROL_CACHE_CONTROL)];
+const STATIC_ASSET_PREFIXES: &[&str] = &[
+ "/avatars/",
+ "/badges/",
+ "/desktop/",
+ "/embeds/",
+ "/emoji/",
+ "/libs/",
+ "/marketing/",
+ "/web/",
+];
fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> {
STATIC_ROOT_FILES
@@ -50,6 +69,12 @@ fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> {
.map(|(_, cache_control)| *cache_control)
}
+fn is_static_asset_path(request_path: &str) -> bool {
+ STATIC_ASSET_PREFIXES
+ .iter()
+ .any(|prefix| request_path.starts_with(prefix))
+}
+
async fn serve_static_file(
static_dir: &str,
request_path: &str,
diff --git a/fluxer_docs/serve.sh b/fluxer_docs/serve.sh
index 8cb5c270a..d744c9067 100755
--- a/fluxer_docs/serve.sh
+++ b/fluxer_docs/serve.sh
@@ -3,6 +3,7 @@ set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VENV="$HERE/.venv"
+REQUIREMENTS_STAMP="$VENV/.requirements.sha256"
ADDR="${ZENSICAL_DEV_ADDR:-0.0.0.0:8000}"
LOG="${ZENSICAL_LOG:-/tmp/zensical-serve.log}"
PORT="${ADDR##*:}"
@@ -14,29 +15,34 @@ ensure_env() {
python3 -m venv "$VENV"
"$VENV/bin/python" -m pip install --quiet --upgrade pip
fi
- "$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt"
+ python_fingerprint="$(python3 -c 'import platform, sys; print(f"{sys.implementation.cache_tag}:{platform.machine()}")')"
+ requirements_hash="$(printf '%s\0%s\n' "$python_fingerprint" "$(sha256sum "$HERE/requirements.txt" | cut -d' ' -f1)" | sha256sum | cut -d' ' -f1)"
+ if [ ! -f "$REQUIREMENTS_STAMP" ] || [ "$(cat "$REQUIREMENTS_STAMP")" != "$requirements_hash" ]; then
+ "$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt"
+ printf '%s\n' "$requirements_hash" >"$REQUIREMENTS_STAMP"
+ fi
}
case "${1:-serve}" in
- --bootstrap)
- ensure_env
- ;;
- --daemon)
- ensure_env
- if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then
- echo "zensical already serving on ${ADDR}"
- exit 0
- fi
- setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 &
- disown 2>/dev/null || true
- echo "zensical serving on ${ADDR} (logs: ${LOG})"
- ;;
- serve)
- ensure_env
- exec "$VENV/bin/zensical" serve -a "$ADDR"
- ;;
- *)
- ensure_env
- exec "$VENV/bin/zensical" "$@"
- ;;
+--bootstrap)
+ ensure_env
+ ;;
+--daemon)
+ ensure_env
+ if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then
+ echo "zensical already serving on ${ADDR}"
+ exit 0
+ fi
+ setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 &
+ disown 2>/dev/null || true
+ echo "zensical serving on ${ADDR} (logs: ${LOG})"
+ ;;
+serve)
+ ensure_env
+ exec "$VENV/bin/zensical" serve -a "$ADDR"
+ ;;
+*)
+ ensure_env
+ exec "$VENV/bin/zensical" "$@"
+ ;;
esac
diff --git a/fluxer_gateway/Dockerfile b/fluxer_gateway/Dockerfile
index 0e715a505..32ff1da63 100644
--- a/fluxer_gateway/Dockerfile
+++ b/fluxer_gateway/Dockerfile
@@ -12,22 +12,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
git \
curl \
make \
- cmake \
gcc \
g++ \
libc6-dev \
- libssl-dev \
- libuv1-dev \
- zlib1g-dev \
- pkg-config \
gettext-base \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
-RUN curl -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \
+RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \
chmod +x /usr/local/bin/rebar3
-RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \
+RUN curl --retry 5 --retry-delay 2 --retry-all-errors --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \
sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}"
COPY . .
@@ -47,8 +42,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
libstdc++6 \
- libuv1t64 \
- zlib1g \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /usr/src/app/fluxer_gateway/_build/prod/rel/fluxer_gateway .
diff --git a/fluxer_gateway/rebar.config b/fluxer_gateway/rebar.config
index 99a565e7b..de42094df 100644
--- a/fluxer_gateway/rebar.config
+++ b/fluxer_gateway/rebar.config
@@ -6,7 +6,6 @@
{jose, "1.11.10"},
{ezstd, "1.1.0"},
{enats, "1.2.0"},
- {erlcass, "4.1.3"},
{eqwalizer_support,
{git_subdir, "https://github.com/whatsapp/eqwalizer.git",
{ref, "c57aa9e3a05553d1ba66ea647a2177eaa469211b"}, "eqwalizer_support"}}
@@ -32,7 +31,6 @@
{relx, [
{release, {fluxer_gateway, "0.0.0"}, [
fluxer_gateway,
- {erlcass, load},
enats,
sasl
]},
@@ -90,7 +88,7 @@
]},
{plt_apps, all_deps},
{plt_extra_apps, [
- crypto, enats, erlcass, inets, jose, public_key, ranch, ssl
+ crypto, enats, inets, jose, public_key, ranch, ssl
]},
{warnings_file, "dialyzer.ignore-warnings"}
]}.
diff --git a/fluxer_gateway/rebar.lock b/fluxer_gateway/rebar.lock
index 2e2d2b7bf..e193fe0ff 100644
--- a/fluxer_gateway/rebar.lock
+++ b/fluxer_gateway/rebar.lock
@@ -9,7 +9,6 @@
{ref,"c57aa9e3a05553d1ba66ea647a2177eaa469211b"},
"eqwalizer_support"},
0},
- {<<"erlcass">>,{pkg,<<"erlcass">>,<<"4.1.3">>},0},
{<<"ezstd">>,{pkg,<<"ezstd">>,<<"1.1.0">>},0},
{<<"jose">>,{pkg,<<"jose">>,<<"1.11.10">>},0},
{<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.4.0">>},1},
@@ -21,7 +20,6 @@
{<<"cowlib">>, <<"54592074EBBBB92EE4746C8A8846E5605052F29309D3A873468D76CDF932076F">>},
{<<"enats">>, <<"D7459C804013CAFA4AF880B18D446C48890D28D372D62AD66C76187E5779248D">>},
{<<"enats_msg">>, <<"50631124F37D88BE76A91A5B96A6565C5981EBF917CD819F0175CA658A966F43">>},
- {<<"erlcass">>, <<"17AC8F39B0A4703B12A8994663920993E191757831E125CF2C7EFB267DB66E7D">>},
{<<"ezstd">>, <<"D3B483D6ACFADFB65DBA4015371E6D54526DBF3D9EF0941B5ADD8BF5890731F4">>},
{<<"jose">>, <<"A903F5227417BD2A08C8A00A0CBCC458118BE84480955E8D251297A425723F83">>},
{<<"opentelemetry_api">>, <<"63CA1742F92F00059298F478048DFB826F4B20D49534493D6919A0DB39B6DB04">>},
@@ -32,7 +30,6 @@
{<<"cowlib">>, <<"7F478D80D66B747344F0EA7708C187645CFCC08B11AA424632F78E25BF05DB51">>},
{<<"enats">>, <<"20DEB3CB1D3E960194DF8B136C40D2DB085B485BBA5E493B340AB5F9FD2BED22">>},
{<<"enats_msg">>, <<"C4F2139E5144FABC99AFE01B8B016AD9DA278CDDC60857AC5D5AFB0AD1283534">>},
- {<<"erlcass">>, <<"37237C12ABE5745E0F900180D0ADB129634619BD6B2F57995BA020E63DDC6BEC">>},
{<<"ezstd">>, <<"28CFA0ED6CC3922095AD5BA0F23392A1664273358B17184BAA909868361184E7">>},
{<<"jose">>, <<"0D6CD36FF8BA174DB29148FC112B5842186B68A90CE9FC2B3EC3AFE76593E614">>},
{<<"opentelemetry_api">>, <<"3DFBBFAA2C2ED3121C5C483162836C4F9027DEF469C41578AF5EF32589FCFC58">>},
diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl
index 81ec09b4f..64c7b6682 100644
--- a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl
+++ b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl
@@ -59,7 +59,7 @@ env_services_config() ->
-spec env_gateway_config() -> map().
env_gateway_config() ->
- maps:merge(env_gateway_base_config(), env_gateway_hotpatch_config()).
+ env_gateway_base_config().
-spec env_gateway_base_config() -> map().
env_gateway_base_config() ->
@@ -107,36 +107,6 @@ env_gateway_base_config() ->
)
}.
--spec env_gateway_hotpatch_config() -> map().
-env_gateway_hotpatch_config() ->
- #{
- <<"hotpatch_enabled">> => env_bool("FLUXER_GATEWAY_HOTPATCH_ENABLED", false),
- <<"hotpatch_cassandra_hosts">> => env_optional_binary(
- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_HOSTS"
- ),
- <<"hotpatch_cassandra_port">> => env_int(
- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PORT", 9042
- ),
- <<"hotpatch_cassandra_keyspace">> => env_binary(
- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_KEYSPACE", <<"fluxer">>
- ),
- <<"hotpatch_cassandra_username">> => env_optional_binary(
- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_USERNAME"
- ),
- <<"hotpatch_cassandra_password">> => env_optional_binary(
- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PASSWORD"
- ),
- <<"hotpatch_public_keys">> => env_optional_binary(
- "FLUXER_GATEWAY_HOTPATCH_PUBLIC_KEYS"
- ),
- <<"hotpatch_poll_interval_ms">> => env_int(
- "FLUXER_GATEWAY_HOTPATCH_POLL_INTERVAL_MS", 5000
- ),
- <<"hotpatch_startup_sync_timeout_ms">> => env_int(
- "FLUXER_GATEWAY_HOTPATCH_STARTUP_SYNC_TIMEOUT_MS", 30000
- )
- }.
-
-spec env_nats_config() -> map().
env_nats_config() ->
#{
@@ -295,20 +265,6 @@ build_http_config(Service) ->
),
gateway_http_recovery_timeout_ms =>
get_int(Service, <<"gateway_http_recovery_timeout_ms">>, 15000),
- hotpatch_enabled => get_bool(Service, <<"hotpatch_enabled">>, false),
- hotpatch_cassandra_hosts =>
- optional_string(get_optional_binary(Service, <<"hotpatch_cassandra_hosts">>)),
- hotpatch_cassandra_port => get_int(Service, <<"hotpatch_cassandra_port">>, 9042),
- hotpatch_cassandra_keyspace =>
- get_binary(Service, <<"hotpatch_cassandra_keyspace">>, <<"fluxer">>),
- hotpatch_cassandra_username =>
- get_optional_binary(Service, <<"hotpatch_cassandra_username">>),
- hotpatch_cassandra_password =>
- get_optional_binary(Service, <<"hotpatch_cassandra_password">>),
- hotpatch_public_keys => get_optional_binary(Service, <<"hotpatch_public_keys">>),
- hotpatch_poll_interval_ms => get_int(Service, <<"hotpatch_poll_interval_ms">>, 5000),
- hotpatch_startup_sync_timeout_ms =>
- get_int(Service, <<"hotpatch_startup_sync_timeout_ms">>, 30000),
gateway_http_cleanup_interval_ms =>
get_int(Service, <<"gateway_http_cleanup_interval_ms">>, 30000),
gateway_http_cleanup_max_age_ms =>
diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl
index 1bea7e15d..e2b953955 100644
--- a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl
+++ b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl
@@ -38,8 +38,7 @@ common_children() ->
child_spec(gateway_nats_pool, gateway_nats_pool),
child_spec(gateway_event_pause, gateway_event_pause),
child_spec(gateway_concurrency, gateway_concurrency),
- child_spec(gateway_rollout_config, gateway_rollout_config),
- child_spec(gateway_hotpatch_reconciler, gateway_hotpatch_reconciler)
+ child_spec(gateway_rollout_config, gateway_rollout_config)
] ++ cluster_children() ++
[
child_spec(gateway_dispatch_relay, gateway_dispatch_relay),
diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl
deleted file mode 100644
index 401595834..000000000
--- a/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl
+++ /dev/null
@@ -1,292 +0,0 @@
-%% SPDX-License-Identifier: AGPL-3.0-or-later
-
--module(gateway_hotpatch_bundle).
--typing([eqwalizer]).
-
--export([
- compress_term/1,
- decompress_term/1,
- bundle_hash/1,
- signing_payload/1,
- sign/2,
- verify_signature/4,
- parse_public_keys/1,
- decode_signed_event/2
-]).
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
--endif.
-
--define(DOMAIN, <<"fluxer-gateway-hotpatch-v1">>).
--define(ZSTD_LEVEL, 3).
-
--spec compress_term(term()) -> {ok, binary()} | {error, term()}.
-compress_term(Term) ->
- compress_binary(term_to_binary(Term, [deterministic])).
-
--spec decompress_term(binary()) -> {ok, term()} | {error, term()}.
-decompress_term(Compressed) when is_binary(Compressed) ->
- case decompress_binary(Compressed) of
- {ok, Binary} -> decode_term(Binary);
- {error, Reason} -> {error, Reason}
- end.
-
--spec decode_term(binary()) -> {ok, term()} | {error, term()}.
-decode_term(Binary) ->
- try
- {ok, binary_to_term(Binary, [safe])}
- catch
- Class:Reason -> {error, {invalid_term, Class, Reason}}
- end.
-
--spec bundle_hash(binary()) -> binary().
-bundle_hash(CompressedBundle) when is_binary(CompressedBundle) ->
- crypto:hash(sha256, CompressedBundle).
-
--spec signing_payload(binary()) -> binary().
-signing_payload(CompressedBundle) when is_binary(CompressedBundle) ->
- <<?DOMAIN/binary, 0, CompressedBundle/binary>>.
-
--spec sign(binary(), binary()) -> {ok, binary()} | {error, term()}.
-sign(CompressedBundle, PrivateKey) when is_binary(CompressedBundle), is_binary(PrivateKey) ->
- try
- {ok, crypto:sign(eddsa, none, signing_payload(CompressedBundle), [PrivateKey, ed25519])}
- catch
- Class:Reason -> {error, {sign_failed, Class, Reason}}
- end.
-
--spec verify_signature(binary(), binary(), binary(), #{binary() => binary()}) ->
- ok | {error, term()}.
-verify_signature(CompressedBundle, SignerKeyId, Signature, PublicKeys) when
- is_binary(CompressedBundle),
- is_binary(SignerKeyId),
- is_binary(Signature),
- is_map(PublicKeys)
-->
- case maps:get(SignerKeyId, PublicKeys, undefined) of
- undefined ->
- {error, {unknown_signer, SignerKeyId}};
- PublicKey ->
- verify_with_key(CompressedBundle, Signature, PublicKey)
- end.
-
--spec parse_public_keys(term()) ->
- {ok, #{binary() => binary()}} | {error, term()}.
-parse_public_keys(undefined) ->
- {ok, #{}};
-parse_public_keys(Value) when is_list(Value) ->
- parse_public_keys(type_conv:ensure_binary(Value));
-parse_public_keys(Value) when is_binary(Value) ->
- Tokens = [
- string:trim(Token)
- || Token <- binary:split(Value, [<<",">>, <<"\n">>, <<";">>], [global]),
- string:trim(Token) =/= <<>>
- ],
- parse_public_key_tokens(Tokens, #{});
-parse_public_keys(Value) ->
- {error, {invalid_public_keys_config, Value}}.
-
--spec decode_signed_event(map(), #{binary() => binary()}) -> {ok, map()} | {error, term()}.
-decode_signed_event(Event, PublicKeys) when is_map(Event), is_map(PublicKeys) ->
- Bundle = event_value(bundle, Event),
- SignerKeyId = event_value(signer_key_id, Event),
- Signature = event_value(signature, Event),
- BundleHash = event_value(bundle_sha256, Event),
- decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys).
-
--spec decode_event_payload(term(), term(), term(), term(), #{binary() => binary()}) ->
- {ok, map()} | {error, term()}.
-decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys) when
- is_binary(Bundle), is_binary(SignerKeyId), is_binary(Signature), is_binary(BundleHash)
-->
- case validate_event_hash(Bundle, BundleHash) of
- ok -> decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys);
- {error, Reason} -> {error, Reason}
- end;
-decode_event_payload(_Bundle, _SignerKeyId, _Signature, _BundleHash, _PublicKeys) ->
- {error, invalid_event_payload}.
-
--spec decode_verified_event(binary(), binary(), binary(), #{binary() => binary()}) ->
- {ok, map()} | {error, term()}.
-decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys) ->
- case verify_signature(Bundle, SignerKeyId, Signature, PublicKeys) of
- ok -> decompress_bundle_map(Bundle);
- {error, Reason} -> {error, Reason}
- end.
-
--spec decompress_bundle_map(binary()) -> {ok, map()} | {error, term()}.
-decompress_bundle_map(Bundle) ->
- case decompress_term(Bundle) of
- {ok, Map} when is_map(Map) -> {ok, Map};
- {ok, Other} -> {error, {invalid_bundle_term, Other}};
- {error, Reason} -> {error, Reason}
- end.
-
--spec compress_binary(binary()) -> {ok, binary()} | {error, term()}.
-compress_binary(Binary) ->
- try erlang:apply(ezstd, compress, [Binary, ?ZSTD_LEVEL]) of
- Compressed when is_binary(Compressed) -> {ok, Compressed};
- {error, Reason} -> {error, {compress_failed, Reason}};
- Other -> {error, {compress_failed, Other}}
- catch
- Class:Reason -> {error, {compress_failed, Class, Reason}}
- end.
-
--spec decompress_binary(binary()) -> {ok, binary()} | {error, term()}.
-decompress_binary(Binary) ->
- try erlang:apply(ezstd, decompress, [Binary]) of
- Decompressed when is_binary(Decompressed) -> {ok, Decompressed};
- Decompressed when is_list(Decompressed) -> {ok, iolist_to_binary(Decompressed)};
- {error, Reason} -> {error, {decompress_failed, Reason}};
- Other -> {error, {decompress_failed, Other}}
- catch
- Class:Reason -> {error, {decompress_failed, Class, Reason}}
- end.
-
--spec verify_with_key(binary(), binary(), binary()) -> ok | {error, term()}.
-verify_with_key(CompressedBundle, Signature, PublicKey) when byte_size(PublicKey) =:= 32 ->
- verify_with_valid_key(CompressedBundle, Signature, PublicKey);
-verify_with_key(_CompressedBundle, _Signature, PublicKey) ->
- {error, {invalid_public_key_size, byte_size(PublicKey)}}.
-
--spec verify_with_valid_key(binary(), binary(), binary()) -> ok | {error, term()}.
-verify_with_valid_key(CompressedBundle, Signature, PublicKey) ->
- try
- crypto:verify(eddsa, none, signing_payload(CompressedBundle), Signature, [
- PublicKey, ed25519
- ])
- of
- true -> ok;
- false -> {error, invalid_signature}
- catch
- Class:Reason -> {error, {verify_failed, Class, Reason}}
- end.
-
--spec parse_public_key_tokens([binary()], #{binary() => binary()}) ->
- {ok, #{binary() => binary()}} | {error, term()}.
-parse_public_key_tokens([], Acc) ->
- {ok, Acc};
-parse_public_key_tokens([Token | Rest], Acc) ->
- case parse_public_key_token(Token) of
- {ok, KeyId, PublicKey} -> parse_public_key_tokens(Rest, Acc#{KeyId => PublicKey});
- {error, Reason} -> {error, Reason}
- end.
-
--spec parse_public_key_token(binary()) -> {ok, binary(), binary()} | {error, term()}.
-parse_public_key_token(Token) ->
- case split_key_token(Token) of
- {ok, KeyId, Encoded} -> parse_public_key_material(KeyId, Encoded);
- error -> {error, {invalid_public_key_token, Token}}
- end.
-
--spec parse_public_key_material(binary(), binary()) ->
- {ok, binary(), binary()} | {error, term()}.
-parse_public_key_material(KeyId, Encoded) ->
- case decode_key_material(Encoded) of
- {ok, PublicKey} when byte_size(PublicKey) =:= 32 -> {ok, KeyId, PublicKey};
- {ok, PublicKey} -> {error, {invalid_public_key_size, KeyId, byte_size(PublicKey)}};
- {error, Reason} -> {error, {invalid_public_key, KeyId, Reason}}
- end.
-
--spec split_key_token(binary()) -> {ok, binary(), binary()} | error.
-split_key_token(Token) ->
- case binary:split(Token, <<":">>) of
- [KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)};
- _ -> split_key_token_equals(Token)
- end.
-
--spec split_key_token_equals(binary()) -> {ok, binary(), binary()} | error.
-split_key_token_equals(Token) ->
- case binary:split(Token, <<"=">>) of
- [KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)};
- _ -> error
- end.
-
--spec decode_key_material(binary()) -> {ok, binary()} | {error, term()}.
-decode_key_material(Encoded) ->
- case try_base64(Encoded) of
- {ok, Decoded} -> {ok, Decoded};
- {error, _} -> try_base64url(Encoded)
- end.
-
--spec try_base64(binary()) -> {ok, binary()} | {error, term()}.
-try_base64(Encoded) ->
- try
- {ok, base64:decode(Encoded)}
- catch
- Class:Reason -> {error, {Class, Reason}}
- end.
-
--spec try_base64url(binary()) -> {ok, binary()} | {error, term()}.
-try_base64url(Encoded) ->
- try
- {ok, base64url:decode(Encoded)}
- catch
- Class:Reason -> {error, {Class, Reason}}
- end.
-
--spec validate_event_hash(binary(), binary()) -> ok | {error, term()}.
-validate_event_hash(Bundle, BundleHash) ->
- case bundle_hash(Bundle) of
- BundleHash -> ok;
- Other -> {error, {bundle_hash_mismatch, Other, BundleHash}}
- end.
-
--spec event_value(atom(), map()) -> term().
-event_value(Key, Event) ->
- maps:get(Key, Event, maps:get(atom_to_binary(Key, utf8), Event, undefined)).
-
--ifdef(TEST).
-
-compress_decompress_roundtrip_test() ->
- Term = #{
- version => 1,
- build_sha => <<"abc123">>,
- modules => [#{module => <<"session_lifecycle">>, expected_current_md5 => <<0:128>>}]
- },
- {ok, Compressed} = compress_term(Term),
- ?assert(is_binary(Compressed)),
- ?assertEqual({ok, Term}, decompress_term(Compressed)).
-
-sign_and_verify_roundtrip_test() ->
- {PublicKey, PrivateKey} = ed25519_keypair(),
- {ok, Compressed} = compress_term(#{version => 1}),
- {ok, Signature} = sign(Compressed, PrivateKey),
- Keys = #{<<"ops">> => PublicKey},
- ?assertEqual(ok, verify_signature(Compressed, <<"ops">>, Signature, Keys)),
- ?assertEqual(
- {error, invalid_signature},
- verify_signature(<<Compressed/binary, 0>>, <<"ops">>, Signature, Keys)
- ).
-
-decode_signed_event_rejects_hash_mismatch_test() ->
- {PublicKey, PrivateKey} = ed25519_keypair(),
- {ok, Compressed} = compress_term(#{version => 1}),
- {ok, Signature} = sign(Compressed, PrivateKey),
- Event = #{
- signer_key_id => <<"ops">>,
- signature => Signature,
- bundle_sha256 => <<0:256>>,
- bundle => Compressed
- },
- ?assertMatch(
- {error, {bundle_hash_mismatch, _, _}},
- decode_signed_event(Event, #{<<"ops">> => PublicKey})
- ).
-
-parse_public_keys_test() ->
- PublicKey = <<1:256>>,
- Encoded = base64:encode(PublicKey),
- ?assertEqual(
- {ok, #{<<"ops">> => PublicKey}}, parse_public_keys(<<"ops:", Encoded/binary>>)
- ).
-
-ed25519_keypair() ->
- {PublicKey, PrivateKey} = crypto:generate_key(eddsa, ed25519),
- {require_binary(PublicKey), require_binary(PrivateKey)}.
-
-require_binary(Value) when is_binary(Value) ->
- Value.
-
--endif.
diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl
deleted file mode 100644
index df8f13c5b..000000000
--- a/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl
+++ /dev/null
@@ -1,459 +0,0 @@
-%% SPDX-License-Identifier: AGPL-3.0-or-later
-
--module(gateway_hotpatch_cli).
--typing([eqwalizer]).
-
--export([main/1, build_bundle/2, sign_bundle/3]).
-
--spec main([string()]) -> no_return().
-main(Args) ->
- halt(run_main(Args)).
-
--spec run_main([string()]) -> non_neg_integer().
-run_main(["bundle", BuildSha | ModuleOrder]) when ModuleOrder =/= [] ->
- write_json_result(build_bundle(type_conv:ensure_binary(BuildSha), ModuleOrder));
-run_main(["sign", SignerKeyId, PrivateKeyPath, BundlePath]) ->
- write_json_result(sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath));
-run_main(["append", BuildSha, CreatedBy, EventPath]) ->
- Result = append_event(
- type_conv:ensure_binary(BuildSha), type_conv:ensure_binary(CreatedBy), EventPath
- ),
- write_append_result(Result);
-run_main(_Args) ->
- write_usage(),
- 64.
-
--spec build_bundle(binary(), [string()]) -> {ok, map()} | {error, term()}.
-build_bundle(BuildSha, ModuleSpecs) when is_binary(BuildSha), is_list(ModuleSpecs) ->
- build_bundle_modules(ModuleSpecs, fun(Modules) ->
- {ok, #{<<"version">> => 1, <<"build_sha">> => BuildSha, <<"modules">> => Modules}}
- end).
-
--spec sign_bundle(binary(), file:filename(), binary()) -> {ok, map()} | {error, term()}.
-sign_bundle(SignerKeyId, PrivateKeyPath, BundleJson) when
- is_binary(SignerKeyId), is_binary(BundleJson)
-->
- case decode_json(BundleJson) of
- {ok, Bundle} -> sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle);
- {error, Reason} -> {error, Reason}
- end.
-
--spec sign_decoded_bundle_with_key(binary(), file:filename(), term()) ->
- {ok, map()} | {error, term()}.
-sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle) ->
- case read_private_key(PrivateKeyPath) of
- {ok, PrivateKey} -> sign_decoded_bundle(SignerKeyId, Bundle, PrivateKey);
- {error, Reason} -> {error, Reason}
- end.
-
--spec sign_bundle_file(string(), file:filename(), file:filename()) ->
- {ok, map()} | {error, term()}.
-sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath) ->
- case file:read_file(BundlePath) of
- {ok, BundleJson} ->
- sign_bundle(type_conv:ensure_binary(SignerKeyId), PrivateKeyPath, BundleJson);
- {error, Reason} ->
- {error, {read_bundle_failed, Reason}}
- end.
-
--spec sign_decoded_bundle(binary(), term(), binary()) -> {ok, map()} | {error, term()}.
-sign_decoded_bundle(SignerKeyId, BundleJson, PrivateKey) ->
- case json_to_bundle(BundleJson) of
- {ok, Bundle} -> sign_bundle_term(SignerKeyId, Bundle, PrivateKey);
- {error, Reason} -> {error, Reason}
- end.
-
--spec sign_bundle_term(binary(), map(), binary()) -> {ok, map()} | {error, term()}.
-sign_bundle_term(SignerKeyId, Bundle, PrivateKey) ->
- case gateway_hotpatch_bundle:compress_term(Bundle) of
- {ok, Compressed} -> sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey);
- {error, Reason} -> {error, Reason}
- end.
-
--spec sign_compressed_bundle(binary(), binary(), binary()) -> {ok, map()} | {error, term()}.
-sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey) ->
- case gateway_hotpatch_bundle:sign(Compressed, PrivateKey) of
- {ok, Signature} -> {ok, signed_event(SignerKeyId, Compressed, Signature)};
- {error, Reason} -> {error, Reason}
- end.
-
--spec signed_event(binary(), binary(), binary()) -> map().
-signed_event(SignerKeyId, Compressed, Signature) ->
- #{
- <<"schema_version">> => 1,
- <<"kind">> => <<"beam_bundle">>,
- <<"created_by">> => gateway_hotpatch_runtime:node_name(),
- <<"signer_key_id">> => SignerKeyId,
- <<"bundle_sha256">> => encode_bytes(gateway_hotpatch_bundle:bundle_hash(Compressed)),
- <<"signature">> => encode_bytes(Signature),
- <<"bundle">> => encode_bytes(Compressed)
- }.
-
--spec append_event(binary(), binary(), file:filename()) -> {ok, binary()} | {error, term()}.
-append_event(BuildSha, CreatedBy, EventPath) ->
- case file:read_file(EventPath) of
- {ok, EventJson} -> append_event_json(BuildSha, CreatedBy, EventJson);
- {error, Reason} -> {error, {read_event_failed, Reason}}
- end.
-
--spec append_event_json(binary(), binary(), binary()) -> {ok, binary()} | {error, term()}.
-append_event_json(BuildSha, CreatedBy, EventJson) ->
- case decode_json(EventJson) of
- {ok, EventJsonTerm} -> append_event_term(BuildSha, CreatedBy, EventJsonTerm);
- {error, Reason} -> {error, Reason}
- end.
-
--spec append_event_term(binary(), binary(), term()) -> {ok, binary()} | {error, term()}.
-append_event_term(BuildSha, CreatedBy, EventJsonTerm) ->
- case json_to_event(EventJsonTerm) of
- {ok, Event} -> append_event_row(BuildSha, CreatedBy, Event);
- {error, Reason} -> {error, Reason}
- end.
-
--spec build_bundle_modules([string()], fun(([map()]) -> {ok, map()})) ->
- {ok, map()} | {error, term()}.
-build_bundle_modules(ModuleSpecs, Fun) ->
- case build_bundle_module_list(ModuleSpecs, []) of
- {ok, Modules} -> Fun(Modules);
- {error, Reason} -> {error, Reason}
- end.
-
--spec build_bundle_module_list([string()], [map()]) -> {ok, [map()]} | {error, term()}.
-build_bundle_module_list([], Acc) ->
- {ok, lists:reverse(Acc)};
-build_bundle_module_list([ModuleSpec | Rest], Acc) ->
- case bundle_module(ModuleSpec) of
- {ok, Module} -> build_bundle_module_list(Rest, [Module | Acc]);
- {error, Reason} -> {error, Reason}
- end.
-
--spec bundle_module(string()) -> {ok, map()} | {error, term()}.
-bundle_module(ModuleSpec) ->
- case split_module_spec(ModuleSpec) of
- {ok, ModuleName, BeamPath0} -> bundle_named_module(ModuleName, BeamPath0);
- {error, Reason} -> {error, Reason}
- end.
-
--spec bundle_named_module(string(), file:filename() | undefined) ->
- {ok, map()} | {error, term()}.
-bundle_named_module(ModuleName, BeamPath0) ->
- case existing_module(ModuleName) of
- {ok, Module} -> bundle_existing_module(Module, BeamPath0);
- {error, Reason} -> {error, Reason}
- end.
-
--spec bundle_existing_module(atom(), file:filename() | undefined) ->
- {ok, map()} | {error, term()}.
-bundle_existing_module(Module, BeamPath0) ->
- case collect_bundle_parts(Module, BeamPath0) of
- {ok, CurrentMd5, TargetMd5, CompressedBeam} ->
- {ok, build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam)};
- {error, Reason} ->
- {error, Reason}
- end.
-
--spec collect_bundle_parts(atom(), file:filename() | undefined) ->
- {ok, binary(), binary(), binary()} | {error, term()}.
-collect_bundle_parts(Module, BeamPath0) ->
- case gateway_hotpatch_loader:current_md5(Module) of
- {ok, CurrentMd5} -> collect_bundle_beam(Module, BeamPath0, CurrentMd5);
- {error, Reason} -> {error, {current_md5_failed, Module, Reason}}
- end.
-
--spec collect_bundle_beam(atom(), file:filename() | undefined, binary()) ->
- {ok, binary(), binary(), binary()} | {error, term()}.
-collect_bundle_beam(Module, BeamPath0, CurrentMd5) ->
- case resolve_beam_path(Module, BeamPath0) of
- {ok, BeamPath} -> read_bundle_beam(Module, BeamPath, CurrentMd5);
- {error, Reason} -> {error, Reason}
- end.
-
--spec read_bundle_beam(atom(), file:filename(), binary()) ->
- {ok, binary(), binary(), binary()} | {error, term()}.
-read_bundle_beam(Module, BeamPath, CurrentMd5) ->
- case file:read_file(BeamPath) of
- {ok, Beam} -> hash_bundle_beam(Module, CurrentMd5, Beam);
- {error, Reason} -> {error, {read_beam_failed, Module, BeamPath, Reason}}
- end.
-
--spec hash_bundle_beam(atom(), binary(), binary()) ->
- {ok, binary(), binary(), binary()} | {error, term()}.
-hash_bundle_beam(Module, CurrentMd5, Beam) ->
- case gateway_hotpatch_loader:beam_md5(Beam) of
- {ok, TargetMd5} -> compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam);
- {error, Reason} -> {error, {target_md5_failed, Module, Reason}}
- end.
-
--spec compress_bundle_beam(atom(), binary(), binary(), binary()) ->
- {ok, binary(), binary(), binary()} | {error, term()}.
-compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam) ->
- case compress_beam(Beam) of
- {ok, CompressedBeam} -> {ok, CurrentMd5, TargetMd5, CompressedBeam};
- {error, Reason} -> {error, {beam_compress_failed, Module, Reason}}
- end.
-
--spec build_module_entry(atom(), binary(), binary(), binary()) -> map().
-build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam) ->
- #{
- <<"module">> => atom_to_binary(Module, utf8),
- <<"expected_current_md5">> => encode_bytes(CurrentMd5),
- <<"target_md5">> => encode_bytes(TargetMd5),
- <<"beam_zstd">> => encode_bytes(CompressedBeam)
- }.
-
--spec append_event_row(binary(), binary(), map()) -> {ok, binary()} | {error, term()}.
-append_event_row(BuildSha, CreatedBy, Event) ->
- _ = fluxer_gateway_env:load(),
- case gateway_hotpatch_store:connect() of
- ok ->
- gateway_hotpatch_store:append_event(
- BuildSha,
- CreatedBy,
- maps:get(signer_key_id, Event),
- maps:get(signature, Event),
- maps:get(bundle_sha256, Event),
- maps:get(bundle, Event)
- );
- {error, Reason} ->
- {error, Reason}
- end.
-
--spec split_module_spec(string()) ->
- {ok, string(), file:filename() | undefined} | {error, term()}.
-split_module_spec(ModuleSpec) ->
- case string:split(ModuleSpec, "=", leading) of
- [ModuleName, BeamPath] when ModuleName =/= "", BeamPath =/= "" ->
- {ok, ModuleName, BeamPath};
- [ModuleName] when ModuleName =/= "" ->
- {ok, ModuleName, undefined};
- _ ->
- {error, {invalid_module_spec, ModuleSpec}}
- end.
-
--spec existing_module(string()) -> {ok, atom()} | {error, term()}.
-existing_module(ModuleName) ->
- try
- {ok, list_to_existing_atom(ModuleName)}
- catch
- error:badarg -> {error, {unknown_module, ModuleName}}
- end.
-
--spec resolve_beam_path(atom(), file:filename() | undefined) ->
- {ok, file:filename()} | {error, term()}.
-resolve_beam_path(Module, undefined) ->
- case code:which(Module) of
- File when is_list(File) -> {ok, File};
- Other -> {error, {module_beam_not_found, Module, Other}}
- end;
-resolve_beam_path(_Module, BeamPath) ->
- {ok, BeamPath}.
-
--spec compress_beam(binary()) -> {ok, binary()} | {error, term()}.
-compress_beam(Beam) ->
- try erlang:apply(ezstd, compress, [Beam, 3]) of
- Compressed when is_binary(Compressed) -> {ok, Compressed};
- {error, Reason} -> {error, Reason};
- Other -> {error, Other}
- catch
- Class:Reason -> {error, {Class, Reason}}
- end.
-
--spec decode_json(binary()) -> {ok, term()} | {error, term()}.
-decode_json(Json) ->
- try
- {ok, json:decode(Json)}
- catch
- Class:Reason -> {error, {decode_json_failed, Class, Reason}}
- end.
-
--spec read_private_key(file:filename()) -> {ok, binary()} | {error, term()}.
-read_private_key(Path) ->
- case file:read_file(Path) of
- {ok, PrivateKey} when byte_size(PrivateKey) =:= 32 ->
- {ok, PrivateKey};
- {ok, PrivateKey} ->
- {error, {invalid_private_key_size, byte_size(PrivateKey)}};
- {error, Reason} ->
- {error, {read_private_key_failed, Reason}}
- end.
-
--spec json_to_bundle(term()) -> {ok, map()} | {error, term()}.
-json_to_bundle(#{
- <<"version">> := Version, <<"build_sha">> := BuildSha, <<"modules">> := Modules
-}) when is_list(Modules) ->
- case json_to_module_entries(Modules, []) of
- {ok, Entries} ->
- {ok, #{
- <<"version">> => Version,
- <<"build_sha">> => BuildSha,
- <<"modules">> => Entries
- }};
- {error, Reason} ->
- {error, Reason}
- end;
-json_to_bundle(Other) ->
- {error, {invalid_bundle_json, Other}}.
-
--spec json_to_module_entries([term()], [map()]) -> {ok, [map()]} | {error, term()}.
-json_to_module_entries([], Acc) ->
- {ok, lists:reverse(Acc)};
-json_to_module_entries([Module | Rest], Acc) ->
- case json_to_module_entry(Module) of
- {ok, Entry} -> json_to_module_entries(Rest, [Entry | Acc]);
- {error, Reason} -> {error, Reason}
- end.
-
--spec json_to_module_entry(term()) -> {ok, map()} | {error, term()}.
-json_to_module_entry(#{
- <<"module">> := Module,
- <<"expected_current_md5">> := ExpectedMd5,
- <<"target_md5">> := TargetMd5,
- <<"beam_zstd">> := BeamZstd
-}) ->
- decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd);
-json_to_module_entry(Other) ->
- {error, {invalid_module_entry_json, Other}}.
-
--spec decode_module_entry(term(), term(), term(), term()) ->
- {ok, map()} | {error, term()}.
-decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) when
- is_binary(Module), is_binary(ExpectedMd5), is_binary(TargetMd5), is_binary(BeamZstd)
-->
- case decode_bytes(ExpectedMd5) of
- {ok, ExpectedMd5Bytes} ->
- decode_module_entry_target(Module, ExpectedMd5Bytes, TargetMd5, BeamZstd);
- {error, Reason} ->
- {error, Reason}
- end;
-decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) ->
- {error, {invalid_module_entry_json, {Module, ExpectedMd5, TargetMd5, BeamZstd}}}.
-
--spec decode_module_entry_target(binary(), binary(), binary(), binary()) ->
- {ok, map()} | {error, term()}.
-decode_module_entry_target(Module, ExpectedMd5, TargetMd5, BeamZstd) ->
- case decode_bytes(TargetMd5) of
- {ok, TargetMd5Bytes} ->
- decode_module_entry_beam(Module, ExpectedMd5, TargetMd5Bytes, BeamZstd);
- {error, Reason} ->
- {error, Reason}
- end.
-
--spec decode_module_entry_beam(binary(), binary(), binary(), binary()) ->
- {ok, map()} | {error, term()}.
-decode_module_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd) ->
- case decode_bytes(BeamZstd) of
- {ok, BeamZstdBytes} ->
- {ok, #{
- <<"module">> => Module,
- <<"expected_current_md5">> => ExpectedMd5,
- <<"target_md5">> => TargetMd5,
- <<"beam_zstd">> => BeamZstdBytes
- }};
- {error, Reason} ->
- {error, Reason}
- end.
-
--spec json_to_event(term()) -> {ok, map()} | {error, term()}.
-json_to_event(#{
- <<"signer_key_id">> := SignerKeyId,
- <<"bundle_sha256">> := BundleSha256,
- <<"signature">> := Signature,
- <<"bundle">> := Bundle
-}) ->
- decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle);
-json_to_event(Other) ->
- {error, {invalid_event_json, Other}}.
-
--spec decode_event_bytes(term(), term(), term(), term()) ->
- {ok, map()} | {error, term()}.
-decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) when
- is_binary(SignerKeyId), is_binary(BundleSha256), is_binary(Signature), is_binary(Bundle)
-->
- case decode_bytes(BundleSha256) of
- {ok, BundleSha256Bytes} ->
- decode_event_signature(SignerKeyId, BundleSha256Bytes, Signature, Bundle);
- {error, Reason} ->
- {error, Reason}
- end;
-decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) ->
- {error, {invalid_event_json, {SignerKeyId, BundleSha256, Signature, Bundle}}}.
-
--spec decode_event_signature(binary(), binary(), binary(), binary()) ->
- {ok, map()} | {error, term()}.
-decode_event_signature(SignerKeyId, BundleSha256, Signature, Bundle) ->
- case decode_bytes(Signature) of
- {ok, SignatureBytes} ->
- decode_event_bundle(SignerKeyId, BundleSha256, SignatureBytes, Bundle);
- {error, Reason} ->
- {error, Reason}
- end.
-
--spec decode_event_bundle(binary(), binary(), binary(), binary()) ->
- {ok, map()} | {error, term()}.
-decode_event_bundle(SignerKeyId, BundleSha256, Signature, Bundle) ->
- case decode_bytes(Bundle) of
- {ok, BundleBytes} ->
- {ok, #{
- signer_key_id => SignerKeyId,
- bundle_sha256 => BundleSha256,
- signature => Signature,
- bundle => BundleBytes
- }};
- {error, Reason} ->
- {error, Reason}
- end.
-
--spec encode_bytes(binary()) -> binary().
-encode_bytes(Binary) ->
- base64:encode(Binary).
-
--spec decode_bytes(binary()) -> {ok, binary()} | {error, term()}.
-decode_bytes(Encoded) when is_binary(Encoded) ->
- try
- {ok, base64:decode(Encoded)}
- catch
- Class:Reason -> {error, {invalid_base64_bytes, Encoded, Class, Reason}}
- end.
-
--spec write_json_result({ok, map()} | {error, term()}) -> non_neg_integer().
-write_json_result({ok, Term}) ->
- write_stdout("~ts~n", [json:encode(Term)]),
- 0;
-write_json_result({error, Reason}) ->
- write_error(Reason).
-
--spec write_append_result({ok, binary()} | {error, term()}) -> non_neg_integer().
-write_append_result({ok, EventId}) ->
- write_stdout("appended hotpatch event ~ts~n", [gateway_hotpatch_loader:hex(EventId)]),
- 0;
-write_append_result({error, Reason}) ->
- write_error(Reason).
-
--spec write_error(term()) -> non_neg_integer().
-write_error(Reason) ->
- write_stderr("gateway hotpatch failed: ~0tp~n", [Reason]),
- 1.
-
--spec write_usage() -> ok.
-write_usage() ->
- write_stderr(
- "usage: gateway_hotpatch_cli bundle BUILD_SHA module_a=/path/to/module_a.beam ...~n"
- " gateway_hotpatch_cli sign SIGNER_KEY_ID PRIVATE_KEY_RAW_FILE bundle.json~n"
- " gateway_hotpatch_cli append BUILD_SHA CREATED_BY signed-event.json~n",
- []
- ).
-
--spec write_stdout(io:format(), [term()]) -> ok.
-write_stdout(Format, Args) ->
- write_stream(standard_io, Format, Args).
-
--spec write_stderr(io:format(), [term()]) -> ok.
-write_stderr(Format, Args) ->
- write_stream(standard_error, Format, Args).
-
--spec write_stream(file:io_device() | standard_io | standard_error, io:format(), [term()]) ->
- ok.
-write_stream(Device, Format, Args) ->
- Output = iolist_to_binary(io_lib:format(Format, Args)),
- _ = file:write(Device, Output),
- ok.
diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl
deleted file mode 100644
index c6b81d49d..000000000
--- a/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl
+++ /dev/null
@@ -1,433 +0,0 @@
-%% SPDX-License-Identifier: AGPL-3.0-or-later
-
--module(gateway_hotpatch_loader).
--typing([eqwalizer]).
-
--export([
- apply_bundle/1,
- current_md5/1,
- beam_module/1,
- beam_md5/1,
- hex/1
-]).
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
--endif.
-
--type module_entry() :: map().
--type apply_result() :: {ok, applied | skipped, atom(), binary()} | {error, term()}.
-
--spec apply_bundle(map()) -> {ok, map()} | {error, term()}.
-apply_bundle(Bundle) when is_map(Bundle) ->
- case
- {
- entry_value(version, Bundle),
- entry_value(build_sha, Bundle),
- entry_value(modules, Bundle)
- }
- of
- {1, BundleBuildSha, Modules} when is_list(Modules) ->
- apply_versioned_bundle(BundleBuildSha, Modules);
- _ ->
- {error, invalid_bundle}
- end;
-apply_bundle(_Bundle) ->
- {error, invalid_bundle}.
-
--spec apply_versioned_bundle(term(), [module_entry()]) -> {ok, map()} | {error, term()}.
-apply_versioned_bundle(BundleBuildSha, Modules) ->
- CurrentBuildSha = gateway_hotpatch_runtime:build_sha(),
- case normalize_binary(BundleBuildSha) of
- CurrentBuildSha -> apply_modules(Modules, #{applied => [], skipped => []});
- Other -> {error, {build_sha_mismatch, Other, CurrentBuildSha}}
- end.
-
--spec current_md5(atom()) -> {ok, binary()} | {error, term()}.
-current_md5(Module) when is_atom(Module) ->
- case hotpatch_loaded_md5(Module) of
- {ok, Md5} -> {ok, Md5};
- error -> current_md5_result(Module, current_md5_from_loaded_or_file(Module))
- end.
-
--spec current_md5_result(atom(), {ok, binary()} | {error, term()}) ->
- {ok, binary()} | {error, term()}.
-current_md5_result(_Module, {ok, Md5}) ->
- {ok, Md5};
-current_md5_result(Module, {error, Reason}) ->
- fallback_current_md5(Module, Reason).
-
--spec fallback_current_md5(atom(), term()) -> {ok, binary()} | {error, term()}.
-fallback_current_md5(Module, Reason) ->
- case hotpatch_loaded_md5(Module) of
- {ok, Md5} -> {ok, Md5};
- error -> {error, Reason}
- end.
-
--spec current_md5_from_loaded_or_file(atom()) -> {ok, binary()} | {error, term()}.
-current_md5_from_loaded_or_file(Module) ->
- case code:get_object_code(Module) of
- {Module, Beam, _File} when is_binary(Beam) -> beam_md5(Beam);
- error -> current_md5_from_file(Module)
- end.
-
--spec beam_module(binary()) -> {ok, atom()} | {error, term()}.
-beam_module(Beam) when is_binary(Beam) ->
- case beam_lib:info(Beam) of
- Info when is_list(Info) -> beam_module_from_info(Info);
- Other -> {error, {beam_info_failed, Other}}
- end.
-
--spec beam_module_from_info(list()) -> {ok, atom()} | {error, term()}.
-beam_module_from_info(Info) ->
- case lists:keyfind(module, 1, Info) of
- {module, Module} when is_atom(Module) -> {ok, Module};
- _ -> {error, module_not_found}
- end.
-
--spec beam_md5(binary()) -> {ok, binary()} | {error, term()}.
-beam_md5(Beam) when is_binary(Beam) ->
- case beam_lib:md5(Beam) of
- {ok, {_Module, Md5}} when is_binary(Md5) -> {ok, Md5};
- Error -> {error, {beam_md5_failed, Error}}
- end.
-
--spec hex(binary()) -> binary().
-hex(Binary) when is_binary(Binary) ->
- iolist_to_binary([[hex_nibble(High), hex_nibble(Low)] || <<High:4, Low:4>> <= Binary]).
-
--spec apply_modules([module_entry()], map()) -> {ok, map()} | {error, term()}.
-apply_modules([], Acc) ->
- {ok, Acc#{module_count => length(maps:get(applied, Acc)) + length(maps:get(skipped, Acc))}};
-apply_modules([Entry | Rest], Acc) ->
- case apply_module(Entry) of
- {ok, applied, Module, TargetMd5} ->
- apply_modules(
- Rest,
- Acc#{applied => [{Module, hex(TargetMd5)} | maps:get(applied, Acc)]}
- );
- {ok, skipped, Module, TargetMd5} ->
- apply_modules(
- Rest,
- Acc#{skipped => [{Module, hex(TargetMd5)} | maps:get(skipped, Acc)]}
- );
- {error, Reason} ->
- {error, Reason}
- end.
-
--spec apply_module(module_entry()) -> apply_result().
-apply_module(Entry) ->
- with_entry(Entry, fun apply_valid_entry/4).
-
--spec apply_valid_entry(atom(), binary(), binary(), binary()) -> apply_result().
-apply_valid_entry(Module, ExpectedMd5, TargetMd5, Beam) ->
- case current_md5(Module) of
- {ok, TargetMd5} ->
- {ok, skipped, Module, TargetMd5};
- {ok, ExpectedMd5} ->
- load_module(Module, Beam, TargetMd5);
- {ok, CurrentMd5} ->
- {error, {md5_mismatch, Module, hex(CurrentMd5), hex(ExpectedMd5)}};
- {error, Reason} ->
- {error, {current_md5_failed, Module, Reason}}
- end.
-
--spec with_entry(module_entry(), fun((atom(), binary(), binary(), binary()) -> apply_result())) ->
- apply_result().
-with_entry(Entry, Fun) when is_map(Entry), is_function(Fun, 4) ->
- case normalize_module(entry_value(module, Entry)) of
- {ok, Module} ->
- ExpectedMd5 = entry_value(expected_current_md5, Entry),
- TargetMd5 = entry_value(target_md5, Entry),
- BeamZstd = entry_value(beam_zstd, Entry),
- with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun);
- {error, Reason} ->
- {error, Reason}
- end;
-with_entry(_Entry, _Fun) ->
- {error, invalid_module_entry}.
-
--spec with_entry_beam(atom(), term(), term(), term(), fun(
- (atom(), binary(), binary(), binary()) -> apply_result()
-)) ->
- apply_result().
-with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun) when
- is_binary(ExpectedMd5),
- is_binary(TargetMd5),
- byte_size(ExpectedMd5) =:= 16,
- byte_size(TargetMd5) =:= 16,
- is_binary(BeamZstd)
-->
- case decompress_beam(BeamZstd) of
- {ok, Beam} -> validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun);
- {error, Reason} -> {error, {beam_decompress_failed, Module, Reason}}
- end;
-with_entry_beam(Module, _ExpectedMd5, _TargetMd5, _BeamZstd, _Fun) ->
- {error, {invalid_module_entry, Module}}.
-
--spec validate_beam(atom(), binary(), binary(), binary(), fun(
- (atom(), binary(), binary(), binary()) -> apply_result()
-)) ->
- apply_result().
-validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun) ->
- case beam_module(Beam) of
- {ok, Module} -> validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun);
- {ok, OtherModule} -> {error, {beam_module_mismatch, Module, OtherModule}};
- {error, Reason} -> {error, {beam_module_failed, Module, Reason}}
- end.
-
--spec validate_beam_md5(atom(), binary(), binary(), binary(), fun(
- (atom(), binary(), binary(), binary()) -> apply_result()
-)) ->
- apply_result().
-validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun) ->
- case beam_md5(Beam) of
- {ok, TargetMd5} ->
- Fun(Module, ExpectedMd5, TargetMd5, Beam);
- {ok, OtherMd5} ->
- {error, {target_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}};
- {error, Reason} ->
- {error, {target_md5_failed, Module, Reason}}
- end.
-
--spec load_module(atom(), binary(), binary()) -> apply_result().
-load_module(Module, Beam, TargetMd5) ->
- case code:soft_purge(Module) of
- true -> load_purged_module(Module, Beam, TargetMd5);
- false -> {error, {soft_purge_failed, Module}}
- end.
-
--spec load_purged_module(atom(), binary(), binary()) -> apply_result().
-load_purged_module(Module, Beam, TargetMd5) ->
- case code:load_binary(Module, atom_to_list(Module) ++ ".beam", Beam) of
- {module, Module} ->
- put_hotpatch_loaded_md5(Module, TargetMd5),
- verify_loaded_module(Module, TargetMd5);
- {error, Reason} ->
- {error, {load_binary_failed, Module, Reason}}
- end.
-
--spec verify_loaded_module(atom(), binary()) -> apply_result().
-verify_loaded_module(Module, TargetMd5) ->
- case current_md5(Module) of
- {ok, TargetMd5} ->
- {ok, applied, Module, TargetMd5};
- {ok, OtherMd5} ->
- {error, {post_load_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}};
- {error, Reason} ->
- {error, {post_load_md5_failed, Module, Reason}}
- end.
-
--spec current_md5_from_file(atom()) -> {ok, binary()} | {error, term()}.
-current_md5_from_file(Module) ->
- case code:which(Module) of
- File when is_list(File) -> current_md5_from_path(Module, File);
- preloaded -> {error, preloaded};
- non_existing -> {error, not_loaded};
- Other -> {error, {not_loadable, Other}}
- end.
-
--spec current_md5_from_path(atom(), file:filename()) -> {ok, binary()} | {error, term()}.
-current_md5_from_path(Module, File) ->
- case beam_lib:md5(File) of
- {ok, {Module, Md5}} when is_binary(Md5) -> {ok, Md5};
- Error -> {error, {beam_file_md5_failed, Error}}
- end.
-
--spec hotpatch_loaded_md5(atom()) -> {ok, binary()} | error.
-hotpatch_loaded_md5(Module) ->
- case persistent_term:get({?MODULE, loaded_md5, Module}, undefined) of
- Md5 when is_binary(Md5), byte_size(Md5) =:= 16 -> {ok, Md5};
- _ -> error
- end.
-
--spec put_hotpatch_loaded_md5(atom(), binary()) -> ok.
-put_hotpatch_loaded_md5(Module, Md5) ->
- persistent_term:put({?MODULE, loaded_md5, Module}, Md5).
-
--spec decompress_beam(binary()) -> {ok, binary()} | {error, term()}.
-decompress_beam(Compressed) ->
- try erlang:apply(ezstd, decompress, [Compressed]) of
- Beam when is_binary(Beam) -> {ok, Beam};
- Beam when is_list(Beam) -> {ok, iolist_to_binary(Beam)};
- {error, Reason} -> {error, Reason};
- Other -> {error, Other}
- catch
- Class:Reason -> {error, {Class, Reason}}
- end.
-
--spec normalize_module(term()) -> {ok, atom()} | {error, term()}.
-normalize_module(Module) when is_atom(Module) ->
- {ok, Module};
-normalize_module(Module) when is_binary(Module) ->
- try
- {ok, binary_to_existing_atom(Module, utf8)}
- catch
- error:badarg -> {error, {unknown_module, Module}}
- end;
-normalize_module(Module) when is_list(Module) ->
- normalize_module(type_conv:ensure_binary(Module));
-normalize_module(Module) ->
- {error, {invalid_module, Module}}.
-
--spec normalize_binary(term()) -> binary().
-normalize_binary(Bin) when is_binary(Bin) -> Bin;
-normalize_binary(List) when is_list(List) -> type_conv:ensure_binary(List);
-normalize_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
-normalize_binary(Other) -> term_to_binary(Other).
-
--spec entry_value(atom(), map()) -> term().
-entry_value(Key, Entry) ->
- maps:get(Key, Entry, maps:get(atom_to_binary(Key, utf8), Entry, undefined)).
-
--spec hex_nibble(0..15) -> integer().
-hex_nibble(N) when N < 10 -> $0 + N;
-hex_nibble(N) -> $a + (N - 10).
-
--ifdef(TEST).
-
-beam_module_and_md5_test() ->
- {module, ?MODULE} = code:ensure_loaded(?MODULE),
- {?MODULE, Binary, _File} = code:get_object_code(?MODULE),
- ?assertEqual({ok, ?MODULE}, beam_module(Binary)),
- {ok, Md5} = beam_md5(Binary),
- ?assertEqual(16, byte_size(Md5)).
-
-apply_bundle_rejects_build_mismatch_test() ->
- Bundle = #{version => 1, build_sha => <<"definitely-not-this-build">>, modules => []},
- ?assertMatch({error, {build_sha_mismatch, _, _}}, apply_bundle(Bundle)).
-
-apply_bundle_accepts_binary_keys_test() ->
- Bundle = #{
- <<"version">> => 1,
- <<"build_sha">> => gateway_hotpatch_runtime:build_sha(),
- <<"modules">> => []
- },
- ?assertEqual(
- {ok, #{applied => [], skipped => [], module_count => 0}}, apply_bundle(Bundle)
- ).
-
-apply_bundle_loads_new_beam_test() ->
- Module = gateway_hotpatch_loader_test_target,
- cleanup_test_module(Module),
- try
- Beam1 = compile_test_module(Module, 1),
- Beam2 = compile_test_module(Module, 2),
- {module, Module} = load_test_beam(Module, Beam1),
- ?assertEqual(1, erlang:apply(Module, version, [])),
- {ok, ExpectedMd5} = current_md5(Module),
- {Entry, TargetMd5} = beam_entry(Module, ExpectedMd5, Beam2),
- Bundle = #{
- version => 1,
- build_sha => gateway_hotpatch_runtime:build_sha(),
- modules => [Entry]
- },
- ?assertEqual(
- {ok, #{applied => [{Module, hex(TargetMd5)}], skipped => [], module_count => 1}},
- apply_bundle(Bundle)
- ),
- ?assertEqual(2, erlang:apply(Module, version, []))
- after
- cleanup_test_module(Module)
- end.
-
-apply_bundle_leaves_prior_module_loaded_when_later_entry_fails_test() ->
- ModuleA = gateway_hotpatch_loader_test_partial_a,
- ModuleB = gateway_hotpatch_loader_test_partial_b,
- cleanup_test_module(ModuleA),
- cleanup_test_module(ModuleB),
- try
- BeamA1 = compile_test_module(ModuleA, 1),
- BeamA2 = compile_test_module(ModuleA, 2),
- BeamB1 = compile_test_module(ModuleB, 1),
- BeamB2 = compile_test_module(ModuleB, 2),
- {module, ModuleA} = load_test_beam(ModuleA, BeamA1),
- {module, ModuleB} = load_test_beam(ModuleB, BeamB1),
- {ok, Md5A1} = current_md5(ModuleA),
- WrongExpectedMd5 = <<0:128>>,
- {EntryA, _Md5A2} = beam_entry(ModuleA, Md5A1, BeamA2),
- {EntryB, _Md5B2} = beam_entry(ModuleB, WrongExpectedMd5, BeamB2),
- Bundle = #{
- version => 1,
- build_sha => gateway_hotpatch_runtime:build_sha(),
- modules => [EntryA, EntryB]
- },
- ?assertMatch({error, {md5_mismatch, ModuleB, _, _}}, apply_bundle(Bundle)),
- ?assertEqual(2, erlang:apply(ModuleA, version, [])),
- ?assertEqual(1, erlang:apply(ModuleB, version, []))
- after
- cleanup_test_module(ModuleA),
- cleanup_test_module(ModuleB)
- end.
-
-hex_test() ->
- ?assertEqual(<<"0001020f10ff">>, hex(<<0, 1, 2, 15, 16, 255>>)).
-
-compile_test_module(Module, Version) ->
- Dir = test_compile_dir(),
- Source = filename:join(Dir, atom_to_list(Module) ++ ".erl"),
- SourceText = io_lib:format(
- "-module(~p).~n-export([version/0]).~nversion() -> ~p.~n",
- [Module, Version]
- ),
- ok = file:write_file(Source, SourceText),
- case compile:file(Source, [binary, return_errors, return_warnings]) of
- {ok, Module, Beam} -> Beam;
- {ok, Module, Beam, _Warnings} -> Beam;
- Error -> erlang:error({test_module_compile_failed, Module, Error})
- end.
-
-test_compile_dir() ->
- Root =
- case os:getenv("TMPDIR") of
- false -> "/tmp";
- Value -> Value
- end,
- Dir = filename:join(Root, "fluxer_hotpatch_loader_tests"),
- ok = filelib:ensure_dir(filename:join(Dir, "placeholder")),
- Dir.
-
-load_test_beam(Module, Beam) ->
- cleanup_test_module(Module),
- BeamPath = filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam"),
- ok = file:write_file(BeamPath, Beam),
- code:load_abs(filename:rootname(BeamPath)).
-
-beam_entry(Module, ExpectedMd5, TargetBeam) ->
- {ok, TargetMd5} = beam_md5(TargetBeam),
- {ok, BeamZstd} = compress_test_beam(TargetBeam),
- {
- #{
- module => Module,
- expected_current_md5 => ExpectedMd5,
- target_md5 => TargetMd5,
- beam_zstd => BeamZstd
- },
- TargetMd5
- }.
-
-compress_test_beam(Beam) ->
- try erlang:apply(ezstd, compress, [Beam, 3]) of
- Compressed when is_binary(Compressed) -> {ok, Compressed};
- {error, Reason} -> {error, Reason};
- Other -> {error, Other}
- catch
- Class:Reason -> {error, {Class, Reason}}
- end.
-
-cleanup_test_module(Module) ->
- _ = code:soft_purge(Module),
- _ = code:purge(Module),
- _ = code:delete(Module),
- _ = code:purge(Module),
- erase_hotpatch_loaded_md5(Module),
- _ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".erl")),
- _ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam")),
- ok.
-
--spec erase_hotpatch_loaded_md5(atom()) -> ok.
-erase_hotpatch_loaded_md5(Module) ->
- _ = persistent_term:erase({?MODULE, loaded_md5, Module}),
- ok.
-
--endif.
diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl
deleted file mode 100644
index 270ccd170..000000000
--- a/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl
+++ /dev/null
@@ -1,319 +0,0 @@
-%% SPDX-License-Identifier: AGPL-3.0-or-later
-
--module(gateway_hotpatch_reconciler).
--typing([eqwalizer]).
-
--behaviour(gen_server).
-
--export([start_link/0, is_ready/0, status/0, reconcile_async/0]).
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
-
--record(state, {
- enabled = false :: boolean(),
- build_sha = <<"dev">> :: binary(),
- public_keys = #{} :: #{binary() => binary()},
- applied_event_ids = [] :: [term()],
- applied_count = 0 :: non_neg_integer(),
- poll_interval_ms = 5000 :: pos_integer(),
- last_error = undefined :: term()
-}).
-
--type state() :: #state{}.
-
--define(SERVER, ?MODULE).
--define(POLL, poll).
--define(STARTUP_RECONCILE_RETRY_MS, 1000).
-
--spec start_link() -> gen_server:start_ret().
-start_link() ->
- gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
-
--spec is_ready() -> boolean().
-is_ready() ->
- gateway_hotpatch_runtime:is_ready().
-
--spec status() -> map().
-status() ->
- gateway_hotpatch_runtime:status().
-
--spec reconcile_async() -> ok.
-reconcile_async() ->
- gen_server:cast(?SERVER, reconcile).
-
--spec init([]) -> {ok, state()} | {stop, term()}.
-init([]) ->
- case gateway_hotpatch_runtime:is_enabled() of
- false -> init_disabled();
- true -> init_enabled()
- end.
-
--spec handle_call(term(), {pid(), term()}, state()) -> {reply, term(), state()}.
-handle_call(status, _From, State) ->
- {reply, state_status(State, gateway_hotpatch_runtime:is_ready()), State};
-handle_call(_Request, _From, State) ->
- {reply, {error, unsupported_call}, State}.
-
--spec handle_cast(term(), state()) -> {noreply, state()}.
-handle_cast(reconcile, State) ->
- {noreply, reconcile_and_publish(State)};
-handle_cast(_Request, State) ->
- {noreply, State}.
-
--spec handle_info(term(), state()) -> {noreply, state()}.
-handle_info(?POLL, #state{enabled = true} = State) ->
- NewState = reconcile_and_publish(State),
- schedule_poll(NewState),
- {noreply, NewState};
-handle_info(_Info, State) ->
- {noreply, State}.
-
--spec terminate(term(), state()) -> ok.
-terminate(_Reason, _State) ->
- ok.
-
--spec code_change(term(), state(), term()) -> {ok, state()}.
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
--spec init_disabled() -> {ok, state()}.
-init_disabled() ->
- BuildSha = gateway_hotpatch_runtime:build_sha(),
- State = #state{enabled = false, build_sha = BuildSha},
- gateway_hotpatch_runtime:put_ready(true),
- publish_status(State, true),
- {ok, State}.
-
--spec init_enabled() -> {ok, state()} | {stop, term()}.
-init_enabled() ->
- gateway_hotpatch_runtime:put_ready(false),
- BuildSha = gateway_hotpatch_runtime:build_sha(),
- PollIntervalMs = runtime_pos_integer(hotpatch_poll_interval_ms, 5000),
- PublicKeyConfig = gateway_hotpatch_runtime:get(hotpatch_public_keys, undefined),
- case gateway_hotpatch_bundle:parse_public_keys(PublicKeyConfig) of
- {ok, PublicKeys} when map_size(PublicKeys) > 0 ->
- State0 = #state{
- enabled = true,
- build_sha = BuildSha,
- public_keys = PublicKeys,
- poll_interval_ms = max(1000, PollIntervalMs)
- },
- publish_status(State0, false),
- init_enabled_connected(State0);
- {ok, _Empty} ->
- init_enabled_config_error(BuildSha, PollIntervalMs, missing_hotpatch_public_keys);
- {error, Reason} ->
- init_enabled_config_error(
- BuildSha, PollIntervalMs, {invalid_hotpatch_public_keys, Reason}
- )
- end.
-
--spec init_enabled_config_error(binary(), pos_integer(), term()) -> {ok, state()}.
-init_enabled_config_error(BuildSha, PollIntervalMs, Reason) ->
- State = #state{
- enabled = true,
- build_sha = BuildSha,
- poll_interval_ms = max(1000, PollIntervalMs),
- last_error = Reason
- },
- gateway_hotpatch_runtime:put_ready(false),
- publish_status(State, false),
- logger:error("Gateway hotpatch configuration invalid: ~0tp", [Reason]),
- schedule_poll(State),
- {ok, State}.
-
--spec init_enabled_connected(state()) -> {ok, state()}.
-init_enabled_connected(State0) ->
- finish_enabled_startup(State0).
-
--spec finish_enabled_startup(state()) -> {ok, state()}.
-finish_enabled_startup(State0) ->
- TimeoutMs = runtime_pos_integer(hotpatch_startup_sync_timeout_ms, 30000),
- DeadlineMs = erlang:monotonic_time(millisecond) + TimeoutMs,
- finish_startup_reconcile(State0, startup_reconcile(State0, DeadlineMs)).
-
--spec finish_startup_reconcile(state(), {ok, state()} | {error, term()}) -> {ok, state()}.
-finish_startup_reconcile(_State0, {ok, State}) ->
- gateway_hotpatch_runtime:put_ready(true),
- publish_status(State, true),
- schedule_poll(State),
- {ok, State};
-finish_startup_reconcile(State0, {error, Reason}) ->
- Error = {hotpatch_startup_sync_failed, Reason},
- State = State0#state{last_error = Error},
- gateway_hotpatch_runtime:put_ready(false),
- publish_status(State, false),
- logger:error("Gateway hotpatch startup sync failed; keeping node unready: ~0tp", [Error]),
- schedule_poll(State),
- {ok, State}.
-
--spec startup_reconcile(state(), integer()) -> {ok, state()} | {error, term()}.
-startup_reconcile(State, DeadlineMs) ->
- case reconcile_once(State) of
- {ok, NewState} ->
- {ok, NewState};
- {error, Reason} ->
- maybe_retry_startup_reconcile(State, DeadlineMs, Reason)
- end.
-
--spec maybe_retry_startup_reconcile(state(), integer(), term()) ->
- {ok, state()} | {error, term()}.
-maybe_retry_startup_reconcile(State, DeadlineMs, Reason) ->
- case erlang:monotonic_time(millisecond) >= DeadlineMs of
- true ->
- {error, Reason};
- false ->
- retry_startup_reconcile(State#state{last_error = Reason}, DeadlineMs, Reason)
- end.
-
--spec retry_startup_reconcile(state(), integer(), term()) -> {ok, state()} | {error, term()}.
-retry_startup_reconcile(State, DeadlineMs, Reason) ->
- case gateway_retry_timer:wait_until(?STARTUP_RECONCILE_RETRY_MS, DeadlineMs) of
- ok -> startup_reconcile(State, DeadlineMs);
- expired -> {error, Reason};
- {error, _InvalidDelay} -> {error, Reason}
- end.
-
--spec reconcile_and_publish(state()) -> state().
-reconcile_and_publish(#state{enabled = false} = State) ->
- gateway_hotpatch_runtime:put_ready(true),
- publish_status(State, true),
- State;
-reconcile_and_publish(State) ->
- case reconcile_once(State) of
- {ok, NewState} ->
- gateway_hotpatch_runtime:put_ready(true),
- publish_status(NewState, true),
- NewState;
- {error, Reason} ->
- publish_reconcile_error(State, Reason)
- end.
-
--spec reconcile_once(state()) -> {ok, state()} | {error, term()}.
-reconcile_once(#state{build_sha = BuildSha} = State) ->
- case gateway_hotpatch_store:connect() of
- ok -> fetch_and_apply_events(BuildSha, State);
- {error, Reason} -> {error, {store_connect_failed, Reason}}
- end.
-
--spec fetch_and_apply_events(binary(), state()) -> {ok, state()} | {error, term()}.
-fetch_and_apply_events(BuildSha, State) ->
- case gateway_hotpatch_store:fetch_events(BuildSha) of
- {ok, Events} -> apply_events(Events, State);
- {error, Reason} -> {error, {fetch_events_failed, Reason}}
- end.
-
--spec publish_reconcile_error(state(), term()) -> state().
-publish_reconcile_error(State, {fetch_events_failed, _Reason} = Error) ->
- ErrorState = State#state{last_error = Error},
- Ready = gateway_hotpatch_runtime:is_ready(),
- gateway_hotpatch_runtime:put_ready(Ready),
- publish_status(ErrorState, Ready),
- logger:warning("Gateway hotpatch fetch failed; keeping current readiness: ~0tp", [Error]),
- ErrorState;
-publish_reconcile_error(State, Reason) ->
- ErrorState = State#state{last_error = Reason},
- gateway_hotpatch_runtime:put_ready(false),
- publish_status(ErrorState, false),
- logger:error("Gateway hotpatch reconciliation failed: ~0tp", [Reason]),
- ErrorState.
-
--spec apply_events([map()], state()) -> {ok, state()} | {error, term()}.
-apply_events([], State) ->
- {ok, State#state{last_error = undefined}};
-apply_events([Event | Rest], State) ->
- EventId = maps:get(event_id, Event, undefined),
- case lists:member(EventId, State#state.applied_event_ids) of
- true ->
- apply_events(Rest, State);
- false ->
- apply_new_event(EventId, Event, Rest, State)
- end.
-
--spec apply_new_event(term(), map(), [map()], state()) -> {ok, state()} | {error, term()}.
-apply_new_event(EventId, Event, Rest, State) ->
- case apply_event(Event, State) of
- {ok, NewState} -> apply_events(Rest, NewState);
- {error, Reason} -> {error, {event_apply_failed, EventId, Reason}}
- end.
-
--spec apply_event(map(), state()) -> {ok, state()} | {error, term()}.
-apply_event(Event, #state{public_keys = PublicKeys, build_sha = BuildSha} = State) ->
- EventId = maps:get(event_id, Event, undefined),
- BundleHash = maps:get(bundle_sha256, Event, <<>>),
- _ = code:ensure_loaded(gateway_hotpatch_loader),
- case gateway_hotpatch_bundle:decode_signed_event(Event, PublicKeys) of
- {ok, Bundle} ->
- apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State);
- {error, Reason} ->
- audit_event(BuildSha, EventId, #{bundle_sha256 => BundleHash}, {error, Reason}),
- {error, Reason}
- end.
-
--spec apply_decoded_event(binary(), term(), binary(), map(), state()) ->
- {ok, state()} | {error, term()}.
-apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State) ->
- Summary0 = #{bundle_sha256 => BundleHash},
- case gateway_hotpatch_loader:apply_bundle(Bundle) of
- {ok, Summary} ->
- handle_applied_event(BuildSha, EventId, Summary0, Summary, State);
- {error, Reason} ->
- audit_event(BuildSha, EventId, Summary0, {error, Reason}),
- {error, Reason}
- end.
-
--spec handle_applied_event(binary(), term(), map(), map(), state()) -> {ok, state()}.
-handle_applied_event(BuildSha, EventId, Summary0, Summary, State) ->
- AuditSummary = maps:merge(Summary0, Summary),
- audit_event(BuildSha, EventId, AuditSummary, ok),
- logger:notice("Applied gateway hotpatch event ~0tp summary=~0tp", [EventId, AuditSummary]),
- {ok, State#state{
- applied_event_ids = [EventId | State#state.applied_event_ids],
- applied_count = State#state.applied_count + 1,
- last_error = undefined
- }}.
-
--spec audit_event(binary(), term(), map(), ok | {error, term()}) -> ok.
-audit_event(BuildSha, EventId, Summary, Result) when is_binary(EventId) ->
- case
- gateway_hotpatch_store:audit_applied(
- BuildSha, gateway_hotpatch_runtime:node_name(), EventId, Summary, Result
- )
- of
- ok ->
- ok;
- {error, Reason} ->
- logger:warning("Gateway hotpatch audit write failed: ~0tp", [Reason]),
- ok
- end;
-audit_event(_BuildSha, _EventId, _Summary, _Result) ->
- ok.
-
--spec publish_status(state(), boolean()) -> ok.
-publish_status(State, Ready) ->
- gateway_hotpatch_runtime:put_status(state_status(State, Ready)).
-
--spec state_status(state(), boolean()) -> map().
-state_status(State, Ready) ->
- #{
- enabled => State#state.enabled,
- ready => Ready,
- build_sha => State#state.build_sha,
- applied_count => State#state.applied_count,
- applied_event_count => length(State#state.applied_event_ids),
- last_error => State#state.last_error
- }.
-
--spec schedule_poll(state()) -> ok.
-schedule_poll(#state{poll_interval_ms = PollIntervalMs}) ->
- erlang:send_after(PollIntervalMs, self(), ?POLL),
- ok.
-
--spec runtime_pos_integer(atom(), pos_integer()) -> pos_integer().
-runtime_pos_integer(Key, Default) ->
- normalize_pos_integer(gateway_hotpatch_runtime:get(Key, Default), Default).
-
--spec normalize_pos_integer(term(), pos_integer()) -> pos_integer().
-normalize_pos_integer(Value, _Default) when is_integer(Value), Value > 0 ->
- Value;
-normalize_pos_integer(_Value, Default) ->
- Default.
diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl
deleted file mode 100644
index d878d7b17..000000000
--- a/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl
+++ /dev/null
@@ -1,78 +0,0 @@
-%% SPDX-License-Identifier: AGPL-3.0-or-later
-
--module(gateway_hotpatch_runtime).
--typing([eqwalizer]).
-
--export([
- build_sha/0,
- node_name/0,
- is_enabled/0,
- get/2,
- put_ready/1,
- is_ready/0,
- put_status/1,
- status/0
-]).
-
--define(READY_KEY, {gateway_hotpatch, ready}).
--define(STATUS_KEY, {gateway_hotpatch, status}).
-
--spec build_sha() -> binary().
-build_sha() ->
- case os:getenv("BUILD_SHA") of
- false -> build_version();
- "" -> build_version();
- Value -> type_conv:ensure_binary(Value, <<"dev">>)
- end.
-
--spec build_version() -> binary().
-build_version() ->
- case os:getenv("BUILD_VERSION") of
- false -> <<"dev">>;
- "" -> <<"dev">>;
- Value -> type_conv:ensure_binary(Value, <<"dev">>)
- end.
-
--spec node_name() -> binary().
-node_name() ->
- atom_to_binary(node(), utf8).
-
--spec is_enabled() -> boolean().
-is_enabled() ->
- case fluxer_gateway_env:get(hotpatch_enabled) of
- true -> true;
- _ -> false
- end.
-
--spec get(atom(), term()) -> term().
-get(Key, Default) ->
- case fluxer_gateway_env:get(Key) of
- undefined -> Default;
- Value -> Value
- end.
-
--spec put_ready(boolean()) -> ok.
-put_ready(Ready) when is_boolean(Ready) ->
- persistent_term:put(?READY_KEY, Ready),
- ok.
-
--spec is_ready() -> boolean().
-is_ready() ->
- case persistent_term:get(?READY_KEY, true) of
- false -> false;
- _ -> true
- end.
-
--spec put_status(map()) -> ok.
-put_status(Status) ->
- persistent_term:put(?STATUS_KEY, Status),
- ok.
-
--spec status() -> map().
-status() ->
- persistent_term:get(?STATUS_KEY, #{
- enabled => false,
- ready => true,
- build_sha => build_sha(),
- applied_count => 0
- }).
diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl
deleted file mode 100644
index 7759c70de..000000000
--- a/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl
+++ /dev/null
@@ -1,277 +0,0 @@
-%% SPDX-License-Identifier: AGPL-3.0-or-later
-
--module(gateway_hotpatch_store).
--typing([eqwalizer]).
-
--export([
- connect/0,
- fetch_events/1,
- append_event/5,
- append_event/6,
- audit_applied/5,
- normalize_event_rows/1
-]).
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
--endif.
-
--define(STMT_FETCH_EVENTS, gateway_hotpatch_fetch_events_by_build).
--define(STMT_APPEND_EVENT, gateway_hotpatch_append_event).
--define(STMT_AUDIT_APPLIED, gateway_hotpatch_audit_applied).
-
--spec connect() -> ok | {error, term()}.
-connect() ->
- case gateway_hotpatch_runtime:get(hotpatch_cassandra_hosts, undefined) of
- Hosts when is_list(Hosts), Hosts =/= [] -> connect_hosts(Hosts);
- Hosts when is_binary(Hosts), byte_size(Hosts) > 0 ->
- start_erlcass(binary_to_list(Hosts));
- _ ->
- {error, missing_hotpatch_cassandra_hosts}
- end.
-
--spec connect_hosts(term()) -> ok | {error, term()}.
-connect_hosts(Hosts) ->
- case type_conv:to_list(Hosts) of
- String when is_list(String), String =/= [] -> start_erlcass(String);
- _ -> {error, missing_hotpatch_cassandra_hosts}
- end.
-
--spec fetch_events(binary()) -> {ok, [map()]} | {error, term()}.
-fetch_events(BuildSha) when is_binary(BuildSha) ->
- case erlcass:execute(?STMT_FETCH_EVENTS, [BuildSha]) of
- {ok, _Columns, Rows} -> {ok, normalize_event_rows(Rows)};
- ok -> {ok, []};
- {error, Reason} -> {error, Reason}
- end.
-
--spec append_event(binary(), binary(), binary(), binary(), binary()) ->
- {ok, binary()} | {error, term()}.
-append_event(CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when
- is_binary(CreatedBy),
- is_binary(SignerKeyId),
- is_binary(Signature),
- is_binary(BundleSha256),
- is_binary(Bundle)
-->
- BuildSha = gateway_hotpatch_runtime:build_sha(),
- append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle).
-
--spec append_event(binary(), binary(), binary(), binary(), binary(), binary()) ->
- {ok, binary()} | {error, term()}.
-append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when
- is_binary(BuildSha),
- is_binary(CreatedBy),
- is_binary(SignerKeyId),
- is_binary(Signature),
- is_binary(BundleSha256),
- is_binary(Bundle)
-->
- case erlcass_uuid:gen_time() of
- {ok, EventId} ->
- Params = append_event_params(
- BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle
- ),
- execute_append_event(EventId, Params);
- {error, Reason} ->
- {error, {event_id_failed, Reason}}
- end.
-
--spec append_event_params(
- binary(), binary(), binary(), binary(), binary(), binary(), binary()
-) -> list().
-append_event_params(BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle) ->
- [
- BuildSha,
- EventId,
- 1,
- <<"beam_bundle">>,
- CreatedBy,
- SignerKeyId,
- BundleSha256,
- Signature,
- Bundle
- ].
-
--spec execute_append_event(binary(), list()) -> {ok, binary()} | {error, term()}.
-execute_append_event(EventId, Params) ->
- case erlcass:execute(?STMT_APPEND_EVENT, Params) of
- ok -> {ok, EventId};
- {ok, _Columns, _Rows} -> {ok, EventId};
- {error, Reason} -> {error, Reason}
- end.
-
--spec audit_applied(binary(), binary(), binary(), map(), ok | {error, term()}) ->
- ok | {error, term()}.
-audit_applied(BuildSha, NodeName, EventId, Summary, Result) when
- is_binary(BuildSha), is_binary(NodeName), is_binary(EventId), is_map(Summary)
-->
- ModuleCount = maps:get(module_count, Summary, 0),
- BundleSha256 = maps:get(bundle_sha256, Summary, <<>>),
- Status = audit_status(Result),
- Error = audit_error(Result),
- Params = [BuildSha, NodeName, EventId, ModuleCount, BundleSha256, Status, Error],
- case erlcass:execute(?STMT_AUDIT_APPLIED, Params) of
- ok -> ok;
- {ok, _Columns, _Rows} -> ok;
- {error, Reason} -> {error, Reason}
- end.
-
--spec normalize_event_rows([list()]) -> [map()].
-normalize_event_rows(Rows) when is_list(Rows) ->
- lists:filtermap(fun normalize_event_row/1, Rows).
-
--spec start_erlcass(string()) -> ok | {error, term()}.
-start_erlcass(Hosts) ->
- Keyspace = gateway_hotpatch_runtime:get(hotpatch_cassandra_keyspace, <<"fluxer">>),
- Port = gateway_hotpatch_runtime:get(hotpatch_cassandra_port, 9042),
- Options0 = [
- {contact_points, type_conv:ensure_binary(Hosts)},
- {port, Port},
- {latency_aware_routing, true},
- {token_aware_routing, true},
- {tcp_nodelay, true},
- {tcp_keepalive, {true, 60}},
- {connect_timeout, 5000},
- {request_timeout, 5000},
- {retry_policy, {default, true}},
- {default_consistency_level, 6}
- ],
- case maybe_credentials(Options0) of
- {ok, Options} -> start_erlcass_with_options(Keyspace, Options);
- {error, Reason} -> {error, Reason}
- end.
-
--spec start_erlcass_with_options(binary(), list()) -> ok | {error, term()}.
-start_erlcass_with_options(Keyspace, Options) ->
- application:set_env(erlcass, keyspace, Keyspace),
- application:set_env(erlcass, cluster_options, Options),
- application:set_env(erlcass, log_level, 2),
- case application:ensure_all_started(erlcass) of
- {ok, _Apps} -> prepare_statements();
- {error, {erlcass, {already_started, erlcass}}} -> prepare_statements();
- {error, Reason} -> {error, {erlcass_start_failed, Reason}}
- end.
-
--spec prepare_statements() -> ok | {error, term()}.
-prepare_statements() ->
- Statements = [
- {?STMT_FETCH_EVENTS, <<
- "SELECT event_id, schema_version, kind, created_by, signer_key_id, "
- "bundle_sha256, signature, bundle "
- "FROM gateway_hotpatch_events_by_build WHERE build_sha = ?"
- >>},
- {?STMT_APPEND_EVENT, <<
- "INSERT INTO gateway_hotpatch_events_by_build "
- "(build_sha, event_id, schema_version, kind, created_at, created_by, "
- "signer_key_id, bundle_sha256, signature, bundle) "
- "VALUES (?, ?, ?, ?, toTimestamp(now()), ?, ?, ?, ?, ?)"
- >>},
- {?STMT_AUDIT_APPLIED, <<
- "INSERT INTO gateway_hotpatch_applied_by_node "
- "(build_sha, node_name, event_id, applied_at, module_count, bundle_sha256, "
- "status, error) "
- "VALUES (?, ?, ?, toTimestamp(now()), ?, ?, ?, ?)"
- >>}
- ],
- prepare_statements(Statements).
-
--spec prepare_statements([{atom(), binary()}]) -> ok | {error, term()}.
-prepare_statements([]) ->
- ok;
-prepare_statements([{Id, Query} | Rest]) ->
- case prepare_statement(Id, Query) of
- ok -> prepare_statements(Rest);
- {error, Reason} -> {error, Reason}
- end.
-
--spec prepare_statement(atom(), binary()) -> ok | {error, term()}.
-prepare_statement(Id, Query) ->
- case erlcass:add_prepare_statement(Id, Query) of
- ok -> ok;
- {error, already_exist} -> ok;
- {error, Reason} -> {error, {prepare_failed, Id, Reason}}
- end.
-
--spec maybe_credentials(list()) -> {ok, list()} | {error, term()}.
-maybe_credentials(Options) ->
- Username = gateway_hotpatch_runtime:get(hotpatch_cassandra_username, undefined),
- Password = gateway_hotpatch_runtime:get(hotpatch_cassandra_password, undefined),
- case {Username, Password} of
- {U, P} when is_binary(U), byte_size(U) > 0, is_binary(P), byte_size(P) > 0 ->
- {ok, [{credentials, {U, P}} | Options]};
- _ ->
- {error, missing_hotpatch_cassandra_credentials}
- end.
-
--spec normalize_event_row(list()) -> {true, map()} | false.
-normalize_event_row([
- EventId,
- SchemaVersion,
- Kind,
- CreatedBy,
- SignerKeyId,
- BundleSha256,
- Signature,
- Bundle
-]) when
- is_binary(EventId),
- is_binary(SignerKeyId),
- is_binary(BundleSha256),
- is_binary(Signature),
- is_binary(Bundle)
-->
- {true, #{
- event_id => EventId,
- schema_version => SchemaVersion,
- kind => Kind,
- created_by => CreatedBy,
- signer_key_id => SignerKeyId,
- bundle_sha256 => BundleSha256,
- signature => Signature,
- bundle => Bundle
- }};
-normalize_event_row(_Row) ->
- false.
-
--spec audit_status(ok | {error, term()}) -> binary().
-audit_status(ok) -> <<"ok">>;
-audit_status({error, _Reason}) -> <<"error">>.
-
--spec audit_error(ok | {error, term()}) -> binary().
-audit_error(ok) ->
- <<>>;
-audit_error({error, Reason}) ->
- iolist_to_binary(io_lib:format("~0tp", [Reason])).
-
--ifdef(TEST).
-
-normalize_event_rows_test() ->
- EventId = <<1:128>>,
- Row = [
- EventId,
- 1,
- <<"beam_bundle">>,
- <<"hampus">>,
- <<"ops">>,
- <<2:256>>,
- <<3:512>>,
- <<"bundle">>
- ],
- ?assertEqual(
- [
- #{
- event_id => EventId,
- schema_version => 1,
- kind => <<"beam_bundle">>,
- created_by => <<"hampus">>,
- signer_key_id => <<"ops">>,
- bundle_sha256 => <<2:256>>,
- signature => <<3:512>>,
- bundle => <<"bundle">>
- }
- ],
- normalize_event_rows([Row, [invalid]])
- ).
-
--endif.
diff --git a/fluxer_gateway/src/gateway/gateway_node_router.erl b/fluxer_gateway/src/gateway/gateway_node_router.erl
index a7c2cf1f5..b2374c9f3 100644
--- a/fluxer_gateway/src/gateway/gateway_node_router.erl
+++ b/fluxer_gateway/src/gateway/gateway_node_router.erl
@@ -98,7 +98,7 @@ fallback_owner_for_role(Role) ->
-spec is_ready() -> boolean().
is_ready() ->
- not is_draining() andalso gateway_hotpatch_reconciler:is_ready().
+ not is_draining().
-spec is_draining() -> boolean().
is_draining() ->
diff --git a/fluxer_gateway/src/gateway/metrics_handler.erl b/fluxer_gateway/src/gateway/metrics_handler.erl
index 2c5f3bb4e..12300c1c5 100644
--- a/fluxer_gateway/src/gateway/metrics_handler.erl
+++ b/fluxer_gateway/src/gateway/metrics_handler.erl
@@ -23,7 +23,6 @@ render_metrics() ->
render_cluster_counters(),
render_process_counts(),
render_push_dispatcher_stats(),
- render_hotpatch_metrics(),
render_vm_metrics()
].
@@ -256,33 +255,6 @@ render_push_dispatcher_stats() ->
]
end.
--spec render_hotpatch_metrics() -> iolist().
-render_hotpatch_metrics() ->
- Status = safe_apply_map(fun gateway_hotpatch_reconciler:status/0),
- Ready = maps:get(ready, Status, true),
- Enabled = maps:get(enabled, Status, false),
- AppliedCount = maps:get(applied_count, Status, 0),
- [
- format_metric(
- <<"fluxer_gateway_hotpatch_enabled">>,
- <<"gauge">>,
- <<"Gateway hotpatch reconciler enabled">>,
- bool_metric(Enabled)
- ),
- format_metric(
- <<"fluxer_gateway_hotpatch_ready">>,
- <<"gauge">>,
- <<"Gateway hotpatch reconciliation readiness">>,
- bool_metric(Ready)
- ),
- format_metric(
- <<"fluxer_gateway_hotpatch_applied_events_total">>,
- <<"counter">>,
- <<"Gateway hotpatch events applied on this node">>,
- integer_to_binary(AppliedCount)
- )
- ].
-
-spec render_vm_metrics() -> iolist().
render_vm_metrics() ->
Memory = safe_apply_list(fun erlang:memory/0),
@@ -376,7 +348,3 @@ format_labeled_series(Name, Type, Help, LabelValues) ->
|| {Label, Value} <- LabelValues
]
].
-
--spec bool_metric(term()) -> binary().
-bool_metric(true) -> <<"1">>;
-bool_metric(_) -> <<"0">>.
diff --git a/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl b/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl
index 1b9c46615..ee09298c9 100644
--- a/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl
+++ b/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl
@@ -159,7 +159,7 @@ flush_pending_syncs_clears_empty_batch_test() ->
State = #{?SYNC_BATCH_STATE_KEY => #{pending_list_ids => #{}}},
?assertEqual(#{}, flush_pending_syncs(State)).
-queue_list_sync_drains_pre_hotpatch_pending_batch_test() ->
+queue_list_sync_drains_preexisting_pending_batch_test() ->
Ref = erlang:send_after(60000, self(), ?FLUSH_SYNC_BATCH_MSG),
State = #{
?SYNC_BATCH_STATE_KEY => #{
diff --git a/fluxer_svc/src/postgres.rs b/fluxer_svc/src/postgres.rs
index 391f7410c..f4ea1dd7c 100644
--- a/fluxer_svc/src/postgres.rs
+++ b/fluxer_svc/src/postgres.rs
@@ -11,6 +11,9 @@ use std::str::FromStr;
use tokio_postgres::{Config as PgConfig, Row, config::SslMode, types::ToSql};
use tokio_postgres_rustls::MakeRustlsConnect;
+const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE: i32 = 0x4658_4b56;
+const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT: &str = "120s";
+
#[derive(Clone, Debug)]
pub struct PostgresConfig {
pub url: Option<String>,
@@ -134,8 +137,30 @@ pub async fn ensure_kv_schema(pool: &Pool, kv_table: &str) -> anyhow::Result<()>
let messages_message_index = quote_identifier(&format!("{kv_table}_messages_message_idx"))?;
let message_reactions_message_index =
quote_identifier(&format!("{kv_table}_message_reactions_message_idx"))?;
- let client = pool.get().await?;
- client
+ let mut client = pool.get().await?;
+ let transaction = client
+ .transaction()
+ .await
+ .context("failed to begin Postgres KV schema transaction")?;
+ transaction
+ .query_one(
+ "SELECT set_config('statement_timeout', $1, true)",
+ &[&POSTGRES_KV_SCHEMA_LOCK_TIMEOUT],
+ )
+ .await
+ .context("failed to configure Postgres KV schema lock timeout")?;
+ transaction
+ .query_one(
+ "SELECT pg_advisory_xact_lock($1, hashtext($2))",
+ &[&POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, &kv_table],
+ )
+ .await
+ .context("failed to acquire Postgres KV schema lock")?;
+ transaction
+ .query_one("SELECT set_config('statement_timeout', '0', true)", &[])
+ .await
+ .context("failed to clear Postgres KV schema lock timeout")?;
+ transaction
.batch_execute(&format!(
r#"
CREATE TABLE IF NOT EXISTS {table} (
@@ -162,6 +187,10 @@ DROP INDEX IF EXISTS {old_partition_index};
))
.await
.context("failed to ensure Postgres KV schema")?;
+ transaction
+ .commit()
+ .await
+ .context("failed to commit Postgres KV schema transaction")?;
Ok(())
}
diff --git a/package.json b/package.json
index 1622ff3a5..7e282a45f 100644
--- a/package.json
+++ b/package.json
@@ -13,11 +13,10 @@
"dev:desktop:canary": "cargo run -p fluxer-dev -- desktop canary",
"dev:desktop:package": "cargo run -p fluxer-dev -- desktop package",
"dev:desktop:run": "cargo run -p fluxer-dev -- desktop run",
- "dev:infra:down": "docker compose -f .devcontainer/docker-compose.yml down",
- "dev:infra:restart": "pnpm dev:infra:down && pnpm dev:infra:up",
- "dev:infra:up": "pnpm dev:infra:up:lite",
- "dev:infra:up:full": "docker compose -f .devcontainer/docker-compose.yml up -d postgres cassandra valkey nats livekit elasticsearch meilisearch mailpit",
- "dev:infra:up:lite": "docker compose -f .devcontainer/docker-compose.yml up -d postgres valkey nats livekit meilisearch mailpit",
+ "dev:infra:restart": "pnpm dev:infra:stop && pnpm dev:infra:start",
+ "dev:infra:start": "cargo run -p fluxer-dev -- infra start",
+ "dev:infra:status": "cargo run -p fluxer-dev -- infra status",
+ "dev:infra:stop": "cargo run -p fluxer-dev -- infra stop",
"dev:services": "cargo run -p fluxer-dev -- rust-services",
"dev:tunnel": "cargo run -p fluxer-dev -- dev --cloudflare-tunnel",
"dev:tunnel:configure": "cargo run -p fluxer-dev -- tunnel configure",
diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.ts
index 6c7e5df55..e2165d47f 100644
--- a/packages/config/src/ConfigLoader.ts
+++ b/packages/config/src/ConfigLoader.ts
@@ -195,6 +195,7 @@ function defaultConfig(): MasterConfig {
api_key: '',
api_secret: '',
url: '',
+ internal_url: '',
webhook_url: '',
},
search: {
diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.ts
index 28c719942..629e55c19 100644
--- a/packages/config/src/MasterConfig.ts
+++ b/packages/config/src/MasterConfig.ts
@@ -243,6 +243,7 @@ export interface MasterConfig {
api_key: string;
api_secret: string;
url: string;
+ internal_url: string;
webhook_url: string;
default_region?: {
id: string;
diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts
index adb9175ce..dae1be5af 100644
--- a/packages/config/src/config_loader/EnvironmentOverrides.ts
+++ b/packages/config/src/config_loader/EnvironmentOverrides.ts
@@ -266,6 +266,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_LIVEKIT_API_KEY: {path: ['integrations', 'voice', 'api_key']},
FLUXER_LIVEKIT_API_SECRET: {path: ['integrations', 'voice', 'api_secret']},
FLUXER_LIVEKIT_URL: {path: ['integrations', 'voice', 'url']},
+ FLUXER_LIVEKIT_INTERNAL_URL: {path: ['integrations', 'voice', 'internal_url']},
FLUXER_LIVEKIT_WEBHOOK_URL: {path: ['integrations', 'voice', 'webhook_url']},
FLUXER_LIVEKIT_DEFAULT_REGION: {path: ['integrations', 'voice', 'default_region'], parse: parseEnvValue},
FLUXER_SEARCH_ENGINE: {path: ['integrations', 'search', 'engine']},
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 83c0a9173..97cb49d5f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -7701,8 +7701,8 @@ packages:
resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==}
engines: {node: '>=16'}
- resolution: {integrity: sha512-DDdwPGz99nmIEv216hKSgLD+D4ikHQHjBC/seF98N9CPqRX4M5mSxT9eTV6oyisnJcuzxtZy4n17yKKQYmYQOA==}
+ resolution: {integrity: sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==}
resolution: {integrity: sha512-HritfMGq9V7SuESeSodHvArs0mLuMk7uh+7hQK2lqdvXrvm50aWxb4RPxkK3mPDdsgHjJ427xNRFITMH2ei+Sw==}
@@ -17551,7 +17551,7 @@ snapshots:
dependencies:
browserslist: 4.28.1
- caniuse-lite: 1.0.30001774
+ caniuse-lite: 1.0.30001810
fraction.js: 5.3.4
picocolors: 1.1.1
postcss: 8.5.6
@@ -17666,7 +17666,7 @@ snapshots:
dependencies:
baseline-browser-mapping: 2.10.0
- caniuse-lite: 1.0.30001774
+ caniuse-lite: 1.0.30001810
electron-to-chromium: 1.5.302
node-releases: 2.0.27
update-browserslist-db: 1.2.3([email protected])
@@ -17761,7 +17761,7 @@ snapshots:
dependencies:
diff --git a/tools/ci/src/app_dev_server.rs b/tools/ci/src/app_dev_server.rs
index c63868325..cbf5d92c5 100644
--- a/tools/ci/src/app_dev_server.rs
+++ b/tools/ci/src/app_dev_server.rs
@@ -4,9 +4,11 @@ use crate::app_wasm::resolve_app_dir;
use anyhow::{Context, Result, bail, ensure};
use clap::Args;
use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsStr;
use std::fs;
+use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
@@ -23,11 +25,51 @@ pub struct AppDevServerArgs {
app_dir: Option<PathBuf>,
}
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(untagged)]
+enum InputFingerprint {
+ Sha256 {
+ sha256: String,
+ #[serde(skip)]
+ legacy_timestamp_ms: Option<f64>,
+ },
+ LegacyTimestamp(f64),
+}
+
+impl InputFingerprint {
+ fn matches(&self, current: &Self) -> bool {
+ match (self, current) {
+ (
+ Self::Sha256 { sha256, .. },
+ Self::Sha256 {
+ sha256: current, ..
+ },
+ ) => sha256 == current,
+ (
+ Self::LegacyTimestamp(timestamp),
+ Self::Sha256 {
+ legacy_timestamp_ms: Some(current),
+ ..
+ },
+ ) => timestamp == current,
+ (Self::LegacyTimestamp(timestamp), Self::LegacyTimestamp(current)) => {
+ timestamp == current
+ }
+ _ => false,
+ }
+ }
+
+ fn is_legacy(&self) -> bool {
+ matches!(self, Self::LegacyTimestamp(_))
+ }
+}
+
+type StepInputs = BTreeMap<String, InputFingerprint>;
+
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct StepMetadata {
- last_run: f64,
- inputs: BTreeMap<String, f64>,
+ inputs: StepInputs,
}
type Metadata = BTreeMap<String, StepMetadata>;
@@ -103,20 +145,14 @@ impl AppDevServer {
)
.await?;
- if env_truthy("FLUXER_APP_SKIP_I18N_COMPILE") {
- eprintln!("Skipping pnpm lingui:compile because FLUXER_APP_SKIP_I18N_COMPILE is set.");
- } else {
- self.run_cached_step(
- "lingui",
- gather_lingui_inputs,
- "pnpm lingui:compile",
- |server, shutdown| {
- Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown))
- },
- &mut shutdown_rx,
- )
- .await?;
- }
+ self.run_cached_step(
+ "lingui",
+ gather_lingui_inputs,
+ "pnpm lingui:compile",
+ |server, shutdown| Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown)),
+ &mut shutdown_rx,
+ )
+ .await?;
if *shutdown_rx.borrow() {
return Ok(());
@@ -175,35 +211,50 @@ impl AppDevServer {
shutdown: &mut watch::Receiver<bool>,
) -> Result<()>
where
- G: Fn(&Path) -> Result<BTreeMap<String, f64>>,
+ G: Fn(&Path) -> Result<StepInputs>,
E: for<'a> FnOnce(
&'a AppDevServer,
&'a mut watch::Receiver<bool>,
)
-> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>>,
{
+ if *shutdown.borrow() {
+ return Ok(());
+ }
let inputs = gather_inputs(&self.project_root)?;
if !self.should_run_step(step_name, &inputs) {
+ let needs_upgrade = self
+ .metadata
+ .get(step_name)
+ .is_some_and(|entry| entry.inputs.values().any(InputFingerprint::is_legacy));
+ if needs_upgrade {
+ self.metadata
+ .insert(step_name.to_string(), StepMetadata { inputs });
+ self.save_metadata()?;
+ }
println!("Skipping {label} (no changes detected)");
return Ok(());
}
execute(self, shutdown).await?;
- self.metadata.insert(
- step_name.to_string(),
- StepMetadata {
- last_run: timestamp_ms(SystemTime::now())?,
- inputs,
- },
- );
+ if *shutdown.borrow() {
+ return Ok(());
+ }
+ self.metadata
+ .insert(step_name.to_string(), StepMetadata { inputs });
self.save_metadata()
}
- fn should_run_step(&self, step_name: &str, inputs: &BTreeMap<String, f64>) -> bool {
+ fn should_run_step(&self, step_name: &str, inputs: &StepInputs) -> bool {
let Some(entry) = self.metadata.get(step_name) else {
return true;
};
- &entry.inputs != inputs
+ entry.inputs.len() != inputs.len()
+ || entry.inputs.iter().any(|(path, cached)| {
+ inputs
+ .get(path)
+ .is_none_or(|current| !cached.matches(current))
+ })
}
async fn run_command(
@@ -286,7 +337,7 @@ impl AppDevServer {
}
}
-fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap<String, f64>> {
+fn collect_file_digests(project_root: &Path, paths: &[PathBuf]) -> Result<StepInputs> {
let mut result = BTreeMap::new();
for rel_path in paths {
let absolute_path = project_root.join(rel_path);
@@ -297,16 +348,16 @@ fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap
"Expected {} to be a file when collecting dev server cache inputs.",
rel_path.display()
);
- result.insert(rel_path_key(rel_path), timestamp_ms(metadata.modified()?)?);
+ result.insert(rel_path_key(rel_path), fingerprint_file(&absolute_path)?);
}
Ok(result)
}
-fn collect_directory_stats<P>(
+fn collect_directory_digests<P>(
project_root: &Path,
root_rel: &Path,
predicate: P,
-) -> Result<BTreeMap<String, f64>>
+) -> Result<StepInputs>
where
P: Fn(&str) -> bool,
{
@@ -335,20 +386,47 @@ where
if !predicate(&key) {
continue;
}
- result.insert(key, timestamp_ms(entry.metadata()?.modified()?)?);
+ result.insert(key, fingerprint_file(entry.path())?);
}
Ok(result)
}
+fn hash_file(path: &Path) -> Result<String> {
+ let mut file = fs::File::open(path)
+ .with_context(|| format!("Failed to open {} for hashing", path.display()))?;
+ let mut hasher = Sha256::new();
+ let mut buffer = [0_u8; 64 * 1024];
+ loop {
+ let bytes_read = file
+ .read(&mut buffer)
+ .with_context(|| format!("Failed to read {} for hashing", path.display()))?;
+ if bytes_read == 0 {
+ break;
+ }
+ hasher.update(&buffer[..bytes_read]);
+ }
+ Ok(hex::encode(hasher.finalize()))
+}
+
+fn fingerprint_file(path: &Path) -> Result<InputFingerprint> {
+ let sha256 = hash_file(path)?;
+ let metadata = fs::metadata(path)
+ .with_context(|| format!("Failed to stat {} after hashing", path.display()))?;
+ Ok(InputFingerprint::Sha256 {
+ sha256,
+ legacy_timestamp_ms: Some(timestamp_ms(metadata.modified()?)?),
+ })
+}
+
fn should_walk_entry(path: &Path, skip_dirs: &BTreeSet<&str>) -> bool {
path.file_name()
.and_then(OsStr::to_str)
.is_none_or(|name| !skip_dirs.contains(name))
}
-fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
+fn gather_wasm_inputs(project_root: &Path) -> Result<StepInputs> {
let markdown_parser_rust_dir = PathBuf::from("../packages/markdown_parser/rust");
- let mut inputs = collect_file_stats(
+ let mut inputs = collect_file_digests(
project_root,
&[
PathBuf::from("../tools/ci/Cargo.toml"),
@@ -361,12 +439,12 @@ fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
markdown_parser_rust_dir.join("Cargo.toml"),
],
)?;
- inputs.extend(collect_directory_stats(
+ inputs.extend(collect_directory_digests(
project_root,
Path::new("rust/libfluxcore"),
|path| !path.contains("/target/"),
)?);
- inputs.extend(collect_directory_stats(
+ inputs.extend(collect_directory_digests(
project_root,
&markdown_parser_rust_dir,
|path| !path.contains("/target/"),
@@ -374,15 +452,15 @@ fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
Ok(inputs)
}
-fn gather_color_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
- collect_file_stats(
+fn gather_color_inputs(project_root: &Path) -> Result<StepInputs> {
+ collect_file_digests(
project_root,
&[PathBuf::from("scripts/GenerateColorSystem.ts")],
)
}
-fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
- collect_file_stats(
+fn gather_message_layout_inputs(project_root: &Path) -> Result<StepInputs> {
+ collect_file_digests(
project_root,
&[
PathBuf::from("scripts/GenerateMessageLayoutCss.ts"),
@@ -391,8 +469,8 @@ fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String,
)
}
-fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
- collect_file_stats(
+fn gather_mask_inputs(project_root: &Path) -> Result<StepInputs> {
+ collect_file_digests(
project_root,
&[
PathBuf::from("scripts/GenerateAvatarMasks.ts"),
@@ -401,14 +479,14 @@ fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
)
}
-fn gather_css_module_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
- collect_directory_stats(project_root, Path::new("src"), |path| {
+fn gather_css_module_inputs(project_root: &Path) -> Result<StepInputs> {
+ collect_directory_digests(project_root, Path::new("src"), |path| {
path.ends_with(".module.css")
})
}
-fn gather_lingui_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
- collect_directory_stats(
+fn gather_lingui_inputs(project_root: &Path) -> Result<StepInputs> {
+ collect_directory_digests(
project_root,
Path::new("src/features/i18n/locales"),
|path| path.ends_with(".po"),
@@ -493,12 +571,6 @@ fn timestamp_ms(timestamp: SystemTime) -> Result<f64> {
* 1000.0)
}
-fn env_truthy(name: &str) -> bool {
- std::env::var(name)
- .ok()
- .is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true"))
-}
-
fn display_command(command: &str, args: &[&str]) -> String {
std::iter::once(command.to_string())
.chain(args.iter().map(|arg| quote_arg(arg)))
diff --git a/tools/dev/cassandra_target_schema.json b/tools/dev/cassandra_target_schema.json
index 7aea67521..797962a5e 100644
--- a/tools/dev/cassandra_target_schema.json
+++ b/tools/dev/cassandra_target_schema.json
@@ -9194,92 +9194,6 @@
],
"primary_key": "((guild_id), webhook_id)",
"options": ""
- },
- {
- "name": "gateway_hotpatch_events_by_build",
- "columns": [
- {
- "name": "build_sha",
- "type": "text"
- },
- {
- "name": "event_id",
- "type": "timeuuid"
- },
- {
- "name": "schema_version",
- "type": "int"
- },
- {
- "name": "kind",
- "type": "text"
- },
- {
- "name": "created_at",
- "type": "timestamp"
- },
- {
- "name": "created_by",
- "type": "text"
- },
- {
- "name": "signer_key_id",
- "type": "text"
- },
- {
- "name": "bundle_sha256",
- "type": "blob"
- },
- {
- "name": "signature",
- "type": "blob"
- },
- {
- "name": "bundle",
- "type": "blob"
- }
- ],
- "primary_key": "((build_sha), event_id)",
- "options": "CLUSTERING ORDER BY (event_id ASC)"
- },
- {
- "name": "gateway_hotpatch_applied_by_node",
- "columns": [
- {
- "name": "build_sha",
- "type": "text"
- },
- {
- "name": "node_name",
- "type": "text"
- },
- {
- "name": "event_id",
- "type": "timeuuid"
- },
- {
- "name": "applied_at",
- "type": "timestamp"
- },
- {
- "name": "module_count",
- "type": "int"
- },
- {
- "name": "bundle_sha256",
- "type": "blob"
- },
- {
- "name": "status",
- "type": "text"
- },
- {
- "name": "error",
- "type": "text"
- }
- ],
- "primary_key": "((build_sha, node_name), event_id)",
- "options": "CLUSTERING ORDER BY (event_id ASC)"
}
],
"indexes": [
diff --git a/tools/dev/src/bootstrap.rs b/tools/dev/src/bootstrap.rs
index 41879683c..74a4232d9 100644
--- a/tools/dev/src/bootstrap.rs
+++ b/tools/dev/src/bootstrap.rs
@@ -1,18 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use crate::desktop::install_desktop;
use crate::gateway::setup_gateway_config;
-use crate::object_store::{bootstrap_schema_and_object_store, wait_s3_api};
+use crate::object_store::bootstrap_schema;
use crate::paths::{ensure_state_dirs, ensure_writable_dev_paths};
-use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http, wait_tcp};
+use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http_success, wait_tcp};
use anyhow::Result;
-pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result<()> {
+pub async fn bootstrap(skip_install: bool) -> Result<()> {
ensure_state_dirs()?;
ensure_writable_dev_paths()?;
if !skip_install {
- crate::proc::run(&["corepack", "enable"])?;
- crate::proc::run(&["corepack", "prepare", "[email protected]", "--activate"])?;
run_command(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
@@ -23,13 +20,10 @@ pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result
..RunOptions::default()
},
)?;
- if !skip_desktop_install {
- install_desktop()?;
- }
}
setup_gateway_config()?;
wait_core_infra().await?;
- bootstrap_schema_and_object_store().await?;
+ bootstrap_schema().await?;
println!("Fluxer dev bootstrap complete.");
Ok(())
}
@@ -43,16 +37,11 @@ pub async fn post_start() -> Result<()> {
}
pub async fn wait_core_infra() -> Result<()> {
+ wait_tcp("Postgres", "postgres", 5432, 120).await?;
+ wait_http_success("Meilisearch", "http://meilisearch:7700/health", 120).await?;
wait_tcp("Valkey", "valkey", 6379, 120).await?;
wait_tcp("NATS", "nats", 4222, 120).await?;
wait_tcp("LiveKit", "livekit", 7880, 120).await?;
crate::media_proxy::ensure_dev_object_store(true, 120).await?;
- wait_tcp("SeaweedFS S3", "127.0.0.1", 8333, 120).await?;
- wait_http(
- "SeaweedFS master",
- "http://127.0.0.1:9333/cluster/status",
- 120,
- )
- .await?;
- wait_s3_api(120).await
+ Ok(())
}
diff --git a/tools/dev/src/dev.rs b/tools/dev/src/dev.rs
index 6d5769e88..1dc8c832c 100644
--- a/tools/dev/src/dev.rs
+++ b/tools/dev/src/dev.rs
@@ -1,15 +1,16 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use crate::gateway::setup_gateway_config;
+use crate::gateway::{build_gateway_cluster_nodes, setup_gateway_config};
use crate::manifest::{
- ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_PORT, LOOPBACK_HOST,
- MEDIA_PROXY_PORT, rust_services,
+ ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_GATEWAY_PORTS_ENV,
+ DEV_PROXY_PORT, GATEWAY_PORT, LOOPBACK_HOST, MEDIA_PROXY_PORT, rust_services,
};
use crate::object_store::s3_endpoint;
use crate::paths::{DESKTOP_DIR, ROOT};
use crate::proc::{
- PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command,
- merged_env, restart_budget_exceeded, run_command, wait_http,
+ AwaitOutcome, PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal,
+ await_or_shutdown, configure_process_group, format_command, merged_env, prefix_output,
+ restart_budget_exceeded, run_command_interruptible, wait_http,
};
use anyhow::{Context, Result, bail};
use std::collections::{BTreeMap, VecDeque};
@@ -30,10 +31,10 @@ const DEFAULT_TASKS: &[&str] = &[
"proxy",
"services",
"media",
- "gateway",
"marketing",
"admin",
"api",
+ "gateway-single",
"worker",
"app",
"app-proxy",
@@ -63,7 +64,7 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
"The private marketing project is unavailable. Authorized maintainers can initialize it with:\n {MARKETING_INITIALIZE_COMMAND}"
);
}
- let tasks = task_table_for_availability(marketing_availability)?;
+ let mut tasks = task_table_for_availability(marketing_availability)?;
let selected = if task_names.is_empty() {
DEFAULT_TASKS
.iter()
@@ -88,11 +89,30 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
unknown.join(", ")
);
}
- ensure_js_dependencies_if_needed(&selected)?;
- ensure_object_store_if_needed(&selected).await?;
- wait_for_search_backend_if_needed(&selected).await?;
- setup_gateway_config()?;
+ let gateway_readiness_port = configure_gateway_proxy_ports(&mut tasks, &selected)?;
let mut shutdown = ShutdownSignal::new()?;
+ match ensure_js_dependencies_if_needed(&selected, &mut shutdown).await? {
+ AwaitOutcome::Completed(()) => {}
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev startup...");
+ return Ok(0);
+ }
+ }
+ match await_or_shutdown(&mut shutdown, ensure_object_store_if_needed(&selected)).await {
+ AwaitOutcome::Completed(result) => result?,
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev startup...");
+ return Ok(0);
+ }
+ }
+ match await_or_shutdown(&mut shutdown, wait_for_search_backend_if_needed(&selected)).await {
+ AwaitOutcome::Completed(result) => result?,
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev startup...");
+ return Ok(0);
+ }
+ }
+ setup_gateway_config()?;
let mut processes = Vec::new();
let mut task_names = Vec::new();
let mut task_specs = Vec::new();
@@ -102,29 +122,82 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
let mut next_object_store_check = Instant::now() + object_store_monitor_interval;
let selected_for_readiness = selected.clone();
for name in selected {
- if name == CLOUDFLARE_TUNNEL_TASK
- && let Err(error) =
- wait_for_cloudflare_tunnel_routes(&mut processes, &selected_for_readiness).await
- {
- crate::gateway::stop_processes(&mut processes);
- return Err(error);
+ if name == CLOUDFLARE_TUNNEL_TASK {
+ match await_or_shutdown(
+ &mut shutdown,
+ wait_for_cloudflare_tunnel_routes(&mut processes, &selected_for_readiness),
+ )
+ .await
+ {
+ AwaitOutcome::Completed(Ok(())) => {}
+ AwaitOutcome::Completed(Err(error)) => {
+ crate::gateway::stop_processes(&mut processes);
+ return Err(error);
+ }
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev tasks...");
+ crate::gateway::stop_processes(&mut processes);
+ return Ok(0);
+ }
+ }
}
- let process = start_task(tasks.get(name.as_str()).expect("validated task"))?;
+ let process = match start_task(tasks.get(name.as_str()).expect("validated task")) {
+ Ok(process) => process,
+ Err(error) => {
+ crate::gateway::stop_processes(&mut processes);
+ return Err(error);
+ }
+ };
processes.push(process);
task_names.push(name.clone());
task_specs.push(tasks.get(name.as_str()).expect("validated task").clone());
task_restarts.push(VecDeque::new());
- if name == "services"
- && let Err(error) = wait_for_rust_services(&mut processes).await
- {
- crate::gateway::stop_processes(&mut processes);
- return Err(error);
+ if name == "services" {
+ match await_or_shutdown(&mut shutdown, wait_for_rust_services(&mut processes)).await {
+ AwaitOutcome::Completed(Ok(())) => {}
+ AwaitOutcome::Completed(Err(error)) => {
+ crate::gateway::stop_processes(&mut processes);
+ return Err(error);
+ }
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev tasks...");
+ crate::gateway::stop_processes(&mut processes);
+ return Ok(0);
+ }
+ }
}
- if name == "api"
- && let Err(error) = wait_for_api(&mut processes).await
- {
- crate::gateway::stop_processes(&mut processes);
- return Err(error);
+ if name == "gateway" || name == "gateway-single" {
+ match await_or_shutdown(
+ &mut shutdown,
+ wait_for_gateway(&mut processes, gateway_readiness_port),
+ )
+ .await
+ {
+ AwaitOutcome::Completed(Ok(())) => {}
+ AwaitOutcome::Completed(Err(error)) => {
+ crate::gateway::stop_processes(&mut processes);
+ return Err(error);
+ }
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev tasks...");
+ crate::gateway::stop_processes(&mut processes);
+ return Ok(0);
+ }
+ }
+ }
+ if name == "api" {
+ match await_or_shutdown(&mut shutdown, wait_for_api(&mut processes)).await {
+ AwaitOutcome::Completed(Ok(())) => {}
+ AwaitOutcome::Completed(Err(error)) => {
+ crate::gateway::stop_processes(&mut processes);
+ return Err(error);
+ }
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev tasks...");
+ crate::gateway::stop_processes(&mut processes);
+ return Ok(0);
+ }
+ }
}
}
loop {
@@ -135,9 +208,17 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
return Err(error);
}
if monitor_object_store && Instant::now() >= next_object_store_check {
- if let Err(error) = monitor_object_store_dependency().await {
- crate::gateway::stop_processes(&mut processes);
- return Err(error);
+ match await_or_shutdown(&mut shutdown, monitor_object_store_dependency()).await {
+ AwaitOutcome::Completed(Ok(())) => {}
+ AwaitOutcome::Completed(Err(error)) => {
+ crate::gateway::stop_processes(&mut processes);
+ return Err(error);
+ }
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping dev tasks...");
+ crate::gateway::stop_processes(&mut processes);
+ return Ok(0);
+ }
}
next_object_store_check = Instant::now() + object_store_monitor_interval;
}
@@ -152,6 +233,38 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
}
}
+fn configure_gateway_proxy_ports(
+ tasks: &mut BTreeMap<&'static str, DevTask>,
+ selected: &[String],
+) -> Result<u16> {
+ let cluster_selected = selected.iter().any(|name| name == "gateway");
+ let single_selected = selected.iter().any(|name| name == "gateway-single");
+ if cluster_selected && single_selected {
+ bail!("Select either `gateway` or `gateway-single`, not both");
+ }
+ let ports = if cluster_selected {
+ build_gateway_cluster_nodes()?
+ .into_iter()
+ .filter(|node| node.role == "websocket")
+ .map(|node| node.http_port)
+ .collect::<Vec<_>>()
+ } else {
+ vec![GATEWAY_PORT]
+ };
+ let value = ports
+ .iter()
+ .map(u16::to_string)
+ .collect::<Vec<_>>()
+ .join(",");
+ let proxy = tasks
+ .get_mut("proxy")
+ .context("dev task table is missing the proxy task")?;
+ proxy
+ .env
+ .push((DEV_PROXY_GATEWAY_PORTS_ENV.to_owned(), Some(value)));
+ Ok(*ports.first().expect("gateway proxy ports are non-empty"))
+}
+
async fn ensure_object_store_if_needed(selected: &[String]) -> Result<()> {
if !selected_needs_object_store(selected) {
return Ok(());
@@ -202,6 +315,7 @@ fn restart_exited_tasks(
let Some(status) = process.try_wait()? else {
continue;
};
+ crate::gateway::stop_child_processes(&mut [process]);
if restart_budget_exceeded(&mut task_restarts[index], Instant::now()) {
bail!(
"Dev task {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",
@@ -237,6 +351,10 @@ fn task_table_for_availability(
marketing_availability: MarketingAvailability,
) -> Result<BTreeMap<&'static str, DevTask>> {
let self_tool = self_tool_command()?;
+ let api_dir = ROOT.join("fluxer_api");
+ let api_tsx = api_dir.join("node_modules/.bin/tsx");
+ let app_dir = ROOT.join("fluxer_app");
+ let app_dev_server = ROOT.join("tools/ci/run.sh");
let public_url = public_url();
let marketing_endpoint = std::env::var("FLUXER_MARKETING_ENDPOINT")
.unwrap_or_else(|_| format!("{public_url}/marketing"));
@@ -260,8 +378,13 @@ fn task_table_for_availability(
});
insert(DevTask {
name: "api",
- args: strings(&["pnpm", "--filter", "fluxer_api", "dev"]),
- cwd: ROOT.clone(),
+ args: vec![
+ api_tsx.display().to_string(),
+ "watch".to_owned(),
+ "--clear-screen=false".to_owned(),
+ "src/AppEntrypoint.ts".to_owned(),
+ ],
+ cwd: api_dir.clone(),
env: vec![
(
"FLUXER_S3_PUBLIC_ENDPOINT".to_owned(),
@@ -286,29 +409,26 @@ fn task_table_for_availability(
});
insert(DevTask {
name: "worker",
- args: strings(&[
- "pnpm",
- "--filter",
- "fluxer_api",
- "exec",
- "tsx",
- "watch",
- "--clear-screen=false",
- "src/WorkerEntrypoint.ts",
- ]),
- cwd: ROOT.clone(),
+ args: vec![
+ api_tsx.display().to_string(),
+ "watch".to_owned(),
+ "--clear-screen=false".to_owned(),
+ "src/WorkerEntrypoint.ts".to_owned(),
+ ],
+ cwd: api_dir,
env: Vec::new(),
});
insert(DevTask {
name: "app",
- args: strings(&["pnpm", "--filter", "fluxer_app", "dev"]),
- cwd: ROOT.clone(),
+ args: vec![
+ app_dev_server.display().to_string(),
+ "app-dev-server".to_owned(),
+ ],
+ cwd: app_dir,
env: vec![
+ ("TOKIO_WORKER_THREADS".to_owned(), Some("4".to_owned())),
+ ("RAYON_NUM_THREADS".to_owned(), Some("4".to_owned())),
("FLUXER_APP_DEV_PORT".to_owned(), Some(APP_PORT.to_string())),
- (
- "FLUXER_APP_SKIP_I18N_COMPILE".to_owned(),
- Some("true".to_owned()),
- ),
(
"FLUXER_STATIC_CDN_ENDPOINT".to_owned(),
Some(public_url.clone()),
@@ -584,27 +704,39 @@ async fn wait_for_search_backend_if_needed(selected: &[String]) -> Result<()> {
return Ok(());
}
let env = merged_env(None, true)?;
+ let backend = SearchBackend::from_env(&env)?;
let search_url = env
.get("FLUXER_SEARCH_URL")
- .cloned()
- .unwrap_or_else(|| default_search_url(&env));
- wait_http(search_backend_label(&env), &search_url, 120).await
+ .filter(|url| !url.trim().is_empty())
+ .context("Missing FLUXER_SEARCH_URL for the selected search backend")?;
+ wait_http(backend.label(), search_url, 120).await
}
-fn default_search_url(env: &BTreeMap<String, String>) -> String {
- if env
- .get("FLUXER_SEARCH_ENGINE")
- .is_some_and(|engine| engine == "meilisearch")
- {
- return "http://127.0.0.1:7700".to_owned();
- }
- "http://127.0.0.1:9200".to_owned()
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum SearchBackend {
+ Elasticsearch,
+ Meilisearch,
}
-fn search_backend_label(env: &BTreeMap<String, String>) -> &'static str {
- match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) {
- Some("meilisearch") => "Meilisearch",
- _ => "Elasticsearch",
+impl SearchBackend {
+ fn from_env(env: &BTreeMap<String, String>) -> Result<Self> {
+ match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) {
+ Some("elasticsearch") => Ok(Self::Elasticsearch),
+ Some("meilisearch") => Ok(Self::Meilisearch),
+ Some(engine) => bail!(
+ "Unsupported FLUXER_SEARCH_ENGINE value {engine:?}; expected `meilisearch` or `elasticsearch`"
+ ),
+ None => {
+ bail!("Missing FLUXER_SEARCH_ENGINE; expected `meilisearch` or `elasticsearch`")
+ }
+ }
+ }
+
+ fn label(self) -> &'static str {
+ match self {
+ Self::Elasticsearch => "Elasticsearch",
+ Self::Meilisearch => "Meilisearch",
+ }
}
}
@@ -627,9 +759,12 @@ fn public_url() -> String {
.unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}"))
}
-fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {
+async fn ensure_js_dependencies_if_needed(
+ selected: &[String],
+ shutdown: &mut ShutdownSignal,
+) -> Result<AwaitOutcome<()>> {
if selected_needs_js_dependency_preflight(selected) {
- run_command(
+ match run_command_interruptible(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
env: PNPM_INSTALL_ENV
@@ -638,11 +773,17 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {
.collect(),
..RunOptions::default()
},
- )?;
+ shutdown,
+ )
+ .await?
+ {
+ AwaitOutcome::Completed(_) => {}
+ AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)),
+ }
}
if selected.iter().any(|name| name == "marketing") {
let marketing_dir = ROOT.join("fluxer_marketing");
- run_command(
+ match run_command_interruptible(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
cwd: &marketing_dir,
@@ -652,9 +793,15 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {
.collect(),
..RunOptions::default()
},
- )?;
+ shutdown,
+ )
+ .await?
+ {
+ AwaitOutcome::Completed(_) => {}
+ AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)),
+ }
}
- Ok(())
+ Ok(AwaitOutcome::Completed(()))
}
fn selected_needs_js_dependency_preflight(selected: &[String]) -> bool {
@@ -689,14 +836,7 @@ fn start_task(task: &DevTask) -> Result<Child> {
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
- #[cfg(unix)]
- unsafe {
- use std::os::unix::process::CommandExt;
- command.pre_exec(|| {
- libc::setsid();
- Ok(())
- });
- }
+ configure_process_group(&mut command);
let mut child = command.spawn()?;
if let Some(stdout) = child.stdout.take() {
let label = task.name.to_owned();
@@ -718,13 +858,16 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> {
.len()
.checked_sub(1)
.expect("services process was just pushed");
+ let deadline = Instant::now() + Duration::from_secs(timeout);
for spec in rust_services() {
for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {
- check_startup_processes(processes, services_index, spec.name, mode)?;
- wait_http(
- &format!("Rust service {}:{mode}", spec.name),
+ let name = format!("Rust service {}:{mode}", spec.name);
+ wait_http_with_startup_checks_until(
+ processes,
+ services_index,
+ &name,
&format!("http://{LOOPBACK_HOST}:{port}/_health"),
- timeout,
+ deadline,
)
.await?;
}
@@ -732,6 +875,25 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> {
Ok(())
}
+async fn wait_for_gateway(processes: &mut [Child], port: u16) -> Result<()> {
+ let timeout = std::env::var("FLUXER_DEV_GATEWAY_READY_TIMEOUT")
+ .ok()
+ .and_then(|value| value.parse().ok())
+ .unwrap_or(180);
+ let gateway_index = processes
+ .len()
+ .checked_sub(1)
+ .expect("gateway process was just pushed");
+ wait_http_with_startup_checks(
+ processes,
+ gateway_index,
+ "Gateway",
+ &format!("http://{LOOPBACK_HOST}:{port}/_health/ready"),
+ timeout,
+ )
+ .await
+}
+
async fn wait_for_api(processes: &mut [Child]) -> Result<()> {
let timeout = std::env::var("FLUXER_DEV_API_READY_TIMEOUT")
.ok()
@@ -827,7 +989,7 @@ async fn wait_http_for_dev_tasks(
while Instant::now() < deadline {
check_dev_startup_processes(processes, name)?;
match client.get(url).send().await {
- Ok(response) if response.status().as_u16() < 500 => {
+ Ok(response) if response.status().is_success() => {
println!("{name} is reachable at {url}");
return Ok(());
}
@@ -849,16 +1011,32 @@ async fn wait_http_with_startup_checks(
name: &str,
url: &str,
timeout_secs: u64,
+) -> Result<()> {
+ wait_http_with_startup_checks_until(
+ processes,
+ watched_index,
+ name,
+ url,
+ Instant::now() + Duration::from_secs(timeout_secs),
+ )
+ .await
+}
+
+async fn wait_http_with_startup_checks_until(
+ processes: &mut [Child],
+ watched_index: usize,
+ name: &str,
+ url: &str,
+ deadline: Instant,
) -> Result<()> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
- let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_error = None;
while Instant::now() < deadline {
check_startup_task_processes(processes, watched_index, name)?;
match client.get(url).send().await {
- Ok(response) if response.status().as_u16() < 500 => {
+ Ok(response) if response.status().is_success() => {
println!("{name} is reachable at {url}");
return Ok(());
}
@@ -901,35 +1079,6 @@ fn check_startup_task_processes(
Ok(())
}
-fn check_startup_processes(
- processes: &mut [Child],
- services_index: usize,
- service_name: &str,
- mode: &str,
-) -> Result<()> {
- for (index, process) in processes.iter_mut().enumerate() {
- if let Some(status) = process.try_wait()? {
- let code = status.code().unwrap_or(1);
- if index == services_index {
- bail!(
- "Rust service supervisor exited with status {code} before {service_name}:{mode} became ready"
- );
- }
- bail!(
- "Dev task exited with status {code} before Rust service {service_name}:{mode} became ready"
- );
- }
- }
- Ok(())
-}
-
-fn prefix_output(label: &str, reader: impl std::io::Read) {
- use std::io::{BufRead, BufReader};
- for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
- println!("[{label}] {line}");
- }
-}
-
#[allow(dead_code)]
fn _cwd_is_root(path: &Path) -> bool {
path == ROOT.as_path()
@@ -940,17 +1089,17 @@ mod tests {
use super::*;
#[test]
- fn default_tasks_keep_legacy_order() {
+ fn default_tasks_use_lightweight_gateway() {
assert_eq!(
DEFAULT_TASKS,
&[
"proxy",
"services",
"media",
- "gateway",
"marketing",
"admin",
"api",
+ "gateway-single",
"worker",
"app",
"app-proxy"
diff --git a/tools/dev/src/gateway.rs b/tools/dev/src/gateway.rs
index a06644bb7..db8cf2956 100644
--- a/tools/dev/src/gateway.rs
+++ b/tools/dev/src/gateway.rs
@@ -1,18 +1,20 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use crate::gateway_reload::{
- ArtifactState, changed_artifacts, hot_reload_enabled, hot_reload_modules, snapshot_artifacts,
- spawn_source_watcher,
-};
use crate::paths::{DEV_GATEWAY_DIR, GATEWAY_CONFIG_DIR, ROOT};
use crate::proc::{
- RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,
- restart_budget_exceeded, run_command,
+ AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown,
+ configure_process_group, force_kill_process_group, format_command, merged_env,
+ pending_shutdown, prefix_output, process_group_running, restart_budget_exceeded,
+ run_command_interruptible, stop_process_group, terminate_process_group,
};
use anyhow::{Context, Result, bail};
+use sha2::{Digest, Sha256};
+#[cfg(target_os = "linux")]
+use std::collections::BTreeSet;
use std::collections::VecDeque;
use std::env;
use std::fs;
+use std::io::Read;
use std::net::{SocketAddr, TcpListener};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
@@ -47,9 +49,14 @@ const GATEWAY_COMPILE_COMMAND: &[&str] = &[
"--step",
"compile",
];
+const GATEWAY_DEPENDENCY_INPUTS: &[&str] = &["rebar.config", "rebar.config.script", "rebar.lock"];
+const GATEWAY_DEPENDENCY_INPUT_MAX_BYTES: u64 = 4 * 1024 * 1024;
+const GATEWAY_DEPENDENCY_STAMP_FILE: &str = "rebar-dependencies.sha256";
+const GATEWAY_DEPENDENCY_STAMP_MAX_BYTES: u64 = 128;
const NODE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
const NODE_RESTART_PORT_WAIT: Duration = Duration::from_secs(75);
-const STOP_GRACE_PERIOD: Duration = Duration::from_secs(10);
+const NESTED_STOP_GRACE_PERIOD: Duration = Duration::from_secs(5);
+const ROOT_STOP_GRACE_PERIOD: Duration = Duration::from_secs(15);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GatewayNode {
@@ -128,33 +135,42 @@ fn remove_stale_gateway_config() -> Result<()> {
Ok(())
}
-pub fn run_gateway() -> Result<()> {
+pub async fn run_gateway() -> Result<i32> {
+ let mut shutdown = ShutdownSignal::new()?;
+ let node_names = std::collections::HashSet::from([env::var("FLUXER_ERLANG_NODE_NAME")
+ .unwrap_or_else(|_| "[email protected]".to_owned())]);
+ cleanup_orphaned_gateway_processes(&node_names).await?;
+ if let Some(signal) = pending_shutdown(&mut shutdown).await {
+ println!("Received {signal}; stopping gateway startup...");
+ return Ok(0);
+ }
setup_gateway_config()?;
- compile_gateway()?;
- let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?;
- let env = merged_env(None, true)?;
- println!("$ {}", format_command(&command));
-
- #[cfg(unix)]
- {
- use std::os::unix::process::CommandExt;
- let err = Command::new(&command[0])
- .args(&command[1..])
- .current_dir(gateway_dir())
- .env_clear()
- .envs(env)
- .exec();
- bail!("failed to exec gateway: {err}");
+ match compile_gateway_interruptible(&mut shutdown).await? {
+ AwaitOutcome::Completed(_) => {}
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping gateway startup...");
+ return Ok(0);
+ }
}
- #[cfg(not(unix))]
- {
- let status = Command::new(&command[0])
- .args(&command[1..])
- .current_dir(gateway_dir())
- .env_clear()
- .envs(env)
- .status()?;
- std::process::exit(status.code().unwrap_or(1));
+ let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?;
+ let command_refs = command.iter().map(String::as_str).collect::<Vec<_>>();
+ let working_dir = gateway_dir();
+ let outcome = run_command_interruptible(
+ &command_refs,
+ RunOptions {
+ cwd: &working_dir,
+ ..RunOptions::default()
+ },
+ &mut shutdown,
+ )
+ .await?;
+ stop_idle_epmd();
+ match outcome {
+ AwaitOutcome::Completed(output) => Ok(output.status.code().unwrap_or(1)),
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping gateway...");
+ Ok(0)
+ }
}
}
@@ -166,11 +182,31 @@ struct SupervisedNode {
pub async fn run_gateway_cluster() -> Result<i32> {
let nodes = build_gateway_cluster_nodes()?;
- cleanup_orphaned_gateway_nodes(&nodes).await?;
- wait_for_cluster_ports_available(&nodes).await?;
- setup_gateway_cluster_config(&nodes)?;
- compile_gateway()?;
let mut shutdown = ShutdownSignal::new()?;
+ let node_names = nodes
+ .iter()
+ .map(GatewayNode::erlang_name)
+ .collect::<std::collections::HashSet<_>>();
+ cleanup_orphaned_gateway_processes(&node_names).await?;
+ if let Some(signal) = pending_shutdown(&mut shutdown).await {
+ println!("Received {signal}; stopping gateway startup...");
+ return Ok(0);
+ }
+ match await_or_shutdown(&mut shutdown, wait_for_cluster_ports_available(&nodes)).await {
+ AwaitOutcome::Completed(result) => result?,
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping gateway startup...");
+ return Ok(0);
+ }
+ }
+ setup_gateway_cluster_config(&nodes)?;
+ match compile_gateway_interruptible(&mut shutdown).await? {
+ AwaitOutcome::Completed(_) => {}
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping gateway startup...");
+ return Ok(0);
+ }
+ }
let static_peers = nodes
.iter()
.map(GatewayNode::erlang_name)
@@ -179,57 +215,35 @@ pub async fn run_gateway_cluster() -> Result<i32> {
let mut supervised = Vec::new();
print_gateway_cluster_topology(&nodes);
for node in &nodes {
+ let child = match start_node(node, &static_peers) {
+ Ok(child) => child,
+ Err(error) => {
+ stop_supervised(&mut supervised);
+ return Err(error);
+ }
+ };
supervised.push(SupervisedNode {
node: node.clone(),
- child: start_node(node, &static_peers)?,
+ child,
restarts: VecDeque::new(),
});
}
- let watcher = hot_reload_enabled().then(spawn_source_watcher);
- if watcher.is_some() {
- println!(
- "Gateway hot reload enabled; watching fluxer_gateway sources (set FLUXER_DEV_GATEWAY_HOT_RELOAD=false to disable)"
- );
- }
- let mut compile: Option<(tokio::task::JoinHandle<bool>, ArtifactState)> = None;
- let mut compile_queued = false;
loop {
- if let Err(error) = restart_exited_nodes(&mut supervised, &static_peers).await {
- stop_supervised(&mut supervised);
- return Err(error);
- }
- if let Some(receiver) = &watcher {
- while receiver.try_recv().is_ok() {
- compile_queued = true;
- }
- }
- if compile_queued && compile.is_none() {
- compile_queued = false;
- println!("Gateway sources changed; recompiling for hot reload...");
- let before = snapshot_artifacts();
- compile = Some((
- tokio::task::spawn_blocking(compile_gateway_for_reload),
- before,
- ));
- }
- if compile
- .as_ref()
- .is_some_and(|(handle, _)| handle.is_finished())
+ match await_or_shutdown(
+ &mut shutdown,
+ restart_exited_nodes(&mut supervised, &static_peers),
+ )
+ .await
{
- let (handle, before) = compile.take().expect("compile task present");
- match handle.await {
- Ok(true) => {
- if let Err(error) =
- apply_hot_reload(&mut supervised, &before, &static_peers).await
- {
- stop_supervised(&mut supervised);
- return Err(error);
- }
- }
- Ok(false) => println!(
- "Gateway compile failed; hot reload skipped (fix the errors and save again)"
- ),
- Err(error) => println!("Gateway compile task failed: {error}"),
+ AwaitOutcome::Completed(Ok(())) => {}
+ AwaitOutcome::Completed(Err(error)) => {
+ stop_supervised(&mut supervised);
+ return Err(error);
+ }
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping gateway cluster...");
+ stop_supervised(&mut supervised);
+ return Ok(0);
}
}
tokio::select! {
@@ -243,24 +257,116 @@ pub async fn run_gateway_cluster() -> Result<i32> {
}
}
-fn compile_gateway() -> Result<()> {
- run_command(GATEWAY_COMPILE_COMMAND, RunOptions::default()).map(drop)
+async fn compile_gateway_interruptible(
+ shutdown: &mut ShutdownSignal,
+) -> Result<AwaitOutcome<std::process::Output>> {
+ let dependency_stamp = prepare_gateway_compile()?;
+ let outcome =
+ run_command_interruptible(GATEWAY_COMPILE_COMMAND, RunOptions::default(), shutdown).await?;
+ if matches!(&outcome, AwaitOutcome::Completed(_)) {
+ write_gateway_dependency_stamp(&dependency_stamp)?;
+ }
+ Ok(outcome)
}
-fn compile_gateway_for_reload() -> bool {
- run_command(
- GATEWAY_COMPILE_COMMAND,
- RunOptions {
- check: false,
- ..RunOptions::default()
- },
- )
- .map(|output| output.status.success())
- .unwrap_or(false)
+fn prepare_gateway_compile() -> Result<String> {
+ let dependency_stamp = gateway_dependency_stamp()?;
+ if gateway_dependency_stamp_matches(&dependency_stamp)? {
+ return Ok(dependency_stamp);
+ }
+ let build_dir = gateway_dir().join("_build/default");
+ println!(
+ "Gateway dependency inputs changed; removing stale {}",
+ build_dir.display()
+ );
+ match fs::remove_dir_all(&build_dir) {
+ Ok(()) => {}
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+ Err(error) => {
+ return Err(error).with_context(|| format!("failed to remove {}", build_dir.display()));
+ }
+ }
+ Ok(dependency_stamp)
}
-fn cluster_cookie() -> String {
- env::var("FLUXER_ERLANG_COOKIE").unwrap_or_else(|_| GATEWAY_CLUSTER_COOKIE.to_owned())
+fn gateway_dependency_stamp() -> Result<String> {
+ let mut hasher = Sha256::new();
+ let mut total_bytes = 0_u64;
+ for input in GATEWAY_DEPENDENCY_INPUTS {
+ let path = gateway_dir().join(input);
+ let metadata = fs::metadata(&path)
+ .with_context(|| format!("failed to read metadata for {}", path.display()))?;
+ if !metadata.is_file() {
+ bail!("gateway dependency input is not a file: {}", path.display());
+ }
+ total_bytes = total_bytes
+ .checked_add(metadata.len())
+ .context("gateway dependency input byte count overflowed")?;
+ if total_bytes > GATEWAY_DEPENDENCY_INPUT_MAX_BYTES {
+ bail!(
+ "gateway dependency inputs exceed {} bytes",
+ GATEWAY_DEPENDENCY_INPUT_MAX_BYTES
+ );
+ }
+ let mut bytes = Vec::with_capacity(metadata.len() as usize);
+ fs::File::open(&path)
+ .with_context(|| format!("failed to open {}", path.display()))?
+ .take(metadata.len() + 1)
+ .read_to_end(&mut bytes)
+ .with_context(|| format!("failed to read {}", path.display()))?;
+ if bytes.len() as u64 != metadata.len() {
+ bail!(
+ "gateway dependency input changed while reading: {}",
+ path.display()
+ );
+ }
+ hasher.update((input.len() as u64).to_le_bytes());
+ hasher.update(input.as_bytes());
+ hasher.update(metadata.len().to_le_bytes());
+ hasher.update(&bytes);
+ }
+ Ok(bytes_to_lower_hex(&hasher.finalize()))
+}
+
+fn bytes_to_lower_hex(bytes: &[u8]) -> String {
+ const HEX: &[u8; 16] = b"0123456789abcdef";
+ let mut output = String::with_capacity(bytes.len() * 2);
+ for byte in bytes {
+ output.push(HEX[(byte >> 4) as usize] as char);
+ output.push(HEX[(byte & 0x0f) as usize] as char);
+ }
+ output
+}
+
+fn gateway_dependency_stamp_matches(expected: &str) -> Result<bool> {
+ let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE);
+ let file = match fs::File::open(&path) {
+ Ok(file) => file,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
+ Err(error) => {
+ return Err(error).with_context(|| format!("failed to open {}", path.display()));
+ }
+ };
+ let mut bytes = Vec::new();
+ file.take(GATEWAY_DEPENDENCY_STAMP_MAX_BYTES + 1)
+ .read_to_end(&mut bytes)
+ .with_context(|| format!("failed to read {}", path.display()))?;
+ if bytes.len() as u64 > GATEWAY_DEPENDENCY_STAMP_MAX_BYTES {
+ bail!(
+ "gateway dependency stamp exceeds {} bytes: {}",
+ GATEWAY_DEPENDENCY_STAMP_MAX_BYTES,
+ path.display()
+ );
+ }
+ Ok(bytes == format!("{expected}\n").as_bytes())
+}
+
+fn write_gateway_dependency_stamp(dependency_stamp: &str) -> Result<()> {
+ fs::create_dir_all(DEV_GATEWAY_DIR.as_path())
+ .with_context(|| format!("failed to create {}", DEV_GATEWAY_DIR.display()))?;
+ let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE);
+ fs::write(&path, format!("{dependency_stamp}\n"))
+ .with_context(|| format!("failed to write {}", path.display()))
}
async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: &str) -> Result<()> {
@@ -285,18 +391,7 @@ async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: &
}
async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result<()> {
- if entry.child.try_wait()?.is_none() {
- terminate_process(&mut entry.child);
- let deadline = Instant::now() + NODE_SHUTDOWN_TIMEOUT;
- while entry.child.try_wait()?.is_none() {
- if Instant::now() >= deadline {
- let _ = entry.child.kill();
- let _ = entry.child.wait();
- break;
- }
- sleep(Duration::from_millis(100)).await;
- }
- }
+ stop_process_group(&mut entry.child, NODE_SHUTDOWN_TIMEOUT).await;
wait_for_ports_available_until(
std::slice::from_ref(&entry.node),
Instant::now() + NODE_RESTART_PORT_WAIT,
@@ -306,79 +401,13 @@ async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result<
Ok(())
}
-async fn apply_hot_reload(
- supervised: &mut [SupervisedNode],
- before: &ArtifactState,
- static_peers: &str,
-) -> Result<()> {
- let after = snapshot_artifacts();
- let diff = changed_artifacts(before, &after);
- if diff.nifs_changed {
- println!("Gateway native NIF artifacts changed; rolling restart of all gateway nodes...");
- for entry in supervised.iter_mut() {
- println!("[gateway:{}] restarting for NIF reload", entry.node.name());
- restart_node(entry, static_peers).await?;
- }
- println!("Gateway rolling restart complete");
- return Ok(());
- }
- if diff.modules.is_empty() {
- println!("Gateway compile finished; no module changes to reload");
- return Ok(());
- }
- println!(
- "Hot reloading {} gateway module(s) across {} node(s): {}",
- diff.modules.len(),
- supervised.len(),
- diff.modules.join(", ")
- );
- let nodes = supervised
- .iter()
- .map(|entry| entry.node.clone())
- .collect::<Vec<_>>();
- let modules = diff.modules.clone();
- let cookie = cluster_cookie();
- let outcome =
- tokio::task::spawn_blocking(move || hot_reload_modules(&nodes, &modules, &cookie)).await;
- match outcome {
- Ok(Ok(outcome)) if outcome.failed_nodes.is_empty() => {
- println!(
- "Gateway hot reload complete ({} module(s) live)",
- diff.modules.len()
- );
- }
- Ok(Ok(outcome)) => {
- for entry in supervised.iter_mut() {
- if !outcome.failed_nodes.contains(&entry.node.erlang_name()) {
- continue;
- }
- if entry.child.try_wait()?.is_some() {
- continue;
- }
- println!(
- "[gateway:{}] hot reload failed; restarting node to pick up new code",
- entry.node.name()
- );
- restart_node(entry, static_peers).await?;
- }
- }
- Ok(Err(error)) => {
- println!("Gateway hot reload failed: {error}; rolling restart of all gateway nodes");
- for entry in supervised.iter_mut() {
- restart_node(entry, static_peers).await?;
- }
- }
- Err(error) => println!("Gateway hot reload task failed: {error}"),
- }
- Ok(())
-}
-
fn stop_supervised(supervised: &mut [SupervisedNode]) {
let mut children = supervised
.iter_mut()
.map(|entry| &mut entry.child)
.collect::<Vec<_>>();
stop_child_processes(&mut children);
+ stop_idle_epmd();
}
pub fn build_gateway_cluster_nodes() -> Result<Vec<GatewayNode>> {
@@ -492,18 +521,20 @@ fn can_bind_tcp_port(port: u16) -> bool {
}
#[cfg(target_os = "linux")]
-async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> {
- let leaders = orphaned_gateway_leaders(nodes)?;
+async fn cleanup_orphaned_gateway_processes(
+ node_names: &std::collections::HashSet<String>,
+) -> Result<()> {
+ let leaders = orphaned_gateway_leaders(node_names)?;
if leaders.is_empty() {
return Ok(());
}
let pids = leaders.iter().map(|leader| leader.pid).collect::<Vec<_>>();
println!(
- "Stopping orphaned gateway cluster node process group(s): {}",
+ "Stopping orphaned gateway process group(s): {}",
format_pids(&pids)
);
- let term_failed = signal_process_groups(&pids, libc::SIGTERM);
+ let term_failed = signal_process_groups(&leaders, libc::SIGTERM);
if !term_failed.is_empty() {
println!(
"SIGTERM delivery failed for orphaned gateway pid(s): {}",
@@ -518,7 +549,12 @@ async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> {
return Ok(());
}
if Instant::now() >= deadline {
- let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);
+ let remaining_leaders = leaders
+ .iter()
+ .filter(|leader| remaining.contains(&leader.pid))
+ .cloned()
+ .collect::<Vec<_>>();
+ let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL);
sleep(Duration::from_millis(200)).await;
let survivors = surviving_gateway_group_pids(&leaders)?;
if survivors.is_empty() {
@@ -556,24 +592,95 @@ fn format_pids(pids: &[i32]) -> String {
}
#[cfg(not(target_os = "linux"))]
-async fn cleanup_orphaned_gateway_nodes(_nodes: &[GatewayNode]) -> Result<()> {
+async fn cleanup_orphaned_gateway_processes(
+ _node_names: &std::collections::HashSet<String>,
+) -> Result<()> {
Ok(())
}
#[cfg(target_os = "linux")]
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq)]
struct GatewayLeader {
+ pid: i32,
+ members: Vec<GatewayProcessIdentity>,
+}
+
+#[cfg(target_os = "linux")]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct GatewayProcessIdentity {
pid: i32,
starttime: u64,
}
#[cfg(target_os = "linux")]
-fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result<Vec<GatewayLeader>> {
- let node_names = nodes
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct GatewayProcess {
+ pid: i32,
+ ppid: i32,
+ pgid: i32,
+ state: char,
+ starttime: u64,
+}
+
+#[cfg(target_os = "linux")]
+fn orphaned_gateway_leaders(
+ node_names: &std::collections::HashSet<String>,
+) -> Result<Vec<GatewayLeader>> {
+ let current_exe = std::env::current_exe().context("failed to resolve fluxer-dev executable")?;
+ let processes = gateway_process_snapshot()?;
+ let mut owned_pids = BTreeSet::new();
+ for process in &processes {
+ if process.ppid != 1 || proc_stat_state_is_dead(process.state) {
+ continue;
+ }
+ let args = proc_cmdline(process.pid);
+ let orphaned_supervisor = cmdline_is_gateway_supervisor(&args, &current_exe);
+ let orphaned_node = (cmdline_has_gateway_node(&args, node_names)
+ || cmdline_has_managed_gateway_config(&args))
+ && fs::read_link(format!("/proc/{}/cwd", process.pid))
+ .ok()
+ .as_deref()
+ == Some(gateway_dir().as_path());
+ if orphaned_supervisor || orphaned_node {
+ owned_pids.insert(process.pid);
+ }
+ }
+ loop {
+ let previous_count = owned_pids.len();
+ for process in &processes {
+ if owned_pids.contains(&process.ppid) {
+ owned_pids.insert(process.pid);
+ }
+ }
+ if owned_pids.len() == previous_count {
+ break;
+ }
+ }
+ let group_pids = processes
.iter()
- .map(GatewayNode::erlang_name)
- .collect::<std::collections::HashSet<_>>();
- let mut leaders = Vec::new();
+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1)
+ .map(|process| process.pgid)
+ .collect::<BTreeSet<_>>();
+ let leaders = group_pids
+ .into_iter()
+ .map(|pid| GatewayLeader {
+ pid,
+ members: processes
+ .iter()
+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid)
+ .map(|process| GatewayProcessIdentity {
+ pid: process.pid,
+ starttime: process.starttime,
+ })
+ .collect(),
+ })
+ .collect::<Vec<_>>();
+ Ok(leaders)
+}
+
+#[cfg(target_os = "linux")]
+fn gateway_process_snapshot() -> Result<Vec<GatewayProcess>> {
+ let mut processes = Vec::new();
for entry in fs::read_dir("/proc").context("failed to read /proc")? {
let Ok(entry) = entry else {
continue;
@@ -585,18 +692,39 @@ fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result<Vec<GatewayLeader>>
else {
continue;
};
- if !cmdline_has_gateway_node(&proc_cmdline(pid), &node_names) {
+ let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
continue;
- }
- if proc_parent_pid(pid) != Some(1) {
+ };
+ let Some((state, ppid, pgid, starttime)) = parse_proc_stat_process(&stat) else {
continue;
- }
- if let Some(starttime) = proc_stat_starttime(pid) {
- leaders.push(GatewayLeader { pid, starttime });
- }
+ };
+ processes.push(GatewayProcess {
+ pid,
+ ppid,
+ pgid,
+ state,
+ starttime,
+ });
}
- leaders.sort_unstable_by_key(|leader| leader.pid);
- Ok(leaders)
+ Ok(processes)
+}
+
+#[cfg(target_os = "linux")]
+fn cmdline_is_gateway_supervisor(args: &[String], current_exe: &Path) -> bool {
+ args.first().map(Path::new) == Some(current_exe)
+ && args.get(1).map(String::as_str) == Some("gateway")
+ && match args.get(2).map(String::as_str) {
+ None | Some("cluster" | "single") => args.len() <= 3,
+ Some(_) => false,
+ }
+}
+
+#[cfg(target_os = "linux")]
+fn cmdline_has_managed_gateway_config(args: &[String]) -> bool {
+ args.windows(2).any(|pair| {
+ matches!(pair[0].as_str(), "-config" | "-args_file")
+ && Path::new(&pair[1]).starts_with(DEV_GATEWAY_DIR.as_path())
+ })
}
#[cfg(any(target_os = "linux", test))]
@@ -618,30 +746,6 @@ fn proc_cmdline(pid: i32) -> Vec<String> {
.collect()
}
-#[cfg(target_os = "linux")]
-fn proc_parent_pid(pid: i32) -> Option<i32> {
- fs::read_to_string(format!("/proc/{pid}/status"))
- .ok()?
- .lines()
- .find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())
-}
-
-#[cfg(target_os = "linux")]
-fn process_exists(pid: i32) -> bool {
- assert!(pid > 0);
- match proc_stat_state_and_pgid(pid) {
- Some((state, _pgid)) => !proc_stat_state_is_dead(state),
- None => false,
- }
-}
-
-#[cfg(target_os = "linux")]
-fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {
- assert!(pid > 0);
- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
- parse_proc_stat_state_and_pgid(&stat)
-}
-
#[cfg(any(target_os = "linux", test))]
fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
let (_, after_comm) = stat.rsplit_once(')')?;
@@ -653,10 +757,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
}
#[cfg(target_os = "linux")]
-fn proc_stat_starttime(pid: i32) -> Option<u64> {
- assert!(pid > 0);
- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
- parse_proc_stat_starttime(&stat)
+fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> {
+ let (_, after_comm) = stat.rsplit_once(')')?;
+ let mut fields = after_comm.split_ascii_whitespace();
+ let state = fields.next()?.chars().next()?;
+ let ppid = fields.next()?.parse().ok()?;
+ let pgid = fields.next()?.parse().ok()?;
+ Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?))
}
#[cfg(any(target_os = "linux", test))]
@@ -672,62 +779,43 @@ fn proc_stat_state_is_dead(state: char) -> bool {
#[cfg(target_os = "linux")]
fn surviving_gateway_group_pids(leaders: &[GatewayLeader]) -> Result<Vec<i32>> {
- let leader_pids = leaders
+ Ok(leaders
.iter()
+ .filter(|leader| gateway_group_has_owned_member(leader))
.map(|leader| leader.pid)
- .collect::<std::collections::HashSet<_>>();
- let mut survivors = Vec::new();
- for entry in fs::read_dir("/proc").context("failed to read /proc")? {
- let Ok(entry) = entry else {
- continue;
- };
- let Some(pid) = entry
- .file_name()
- .to_str()
- .and_then(|name| name.parse::<i32>().ok())
- else {
- continue;
- };
- let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
- continue;
+ .collect())
+}
+
+#[cfg(target_os = "linux")]
+fn gateway_group_has_owned_member(leader: &GatewayLeader) -> bool {
+ leader.members.iter().any(|member| {
+ let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else {
+ return false;
};
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
- continue;
+ return false;
};
- if proc_stat_state_is_dead(state) {
- continue;
- }
- if leader_pids.contains(&pgid) {
- survivors.push(pid);
- continue;
- }
- let leader = leaders.iter().find(|leader| leader.pid == pid);
- if let Some(leader) = leader
- && parse_proc_stat_starttime(&stat) == Some(leader.starttime)
- {
- survivors.push(pid);
- }
- }
- survivors.sort_unstable();
- Ok(survivors)
+ !proc_stat_state_is_dead(state)
+ && pgid == leader.pid
+ && parse_proc_stat_starttime(&stat) == Some(member.starttime)
+ })
}
#[cfg(target_os = "linux")]
-fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {
+fn signal_process_groups(leaders: &[GatewayLeader], signal: i32) -> Vec<i32> {
assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);
- let mut failed = Vec::with_capacity(pids.len());
- for pid in pids {
- assert!(*pid > 0);
- let group_result = unsafe { libc::kill(-pid, signal) };
- if group_result == 0 {
+ let mut failed = Vec::with_capacity(leaders.len());
+ for leader in leaders {
+ assert!(leader.pid > 1);
+ if !gateway_group_has_owned_member(leader) {
continue;
}
- let direct_result = unsafe { libc::kill(*pid, signal) };
- if direct_result == 0 {
+ let group_result = unsafe { libc::kill(-leader.pid, signal) };
+ if group_result == 0 {
continue;
}
- if process_exists(*pid) {
- failed.push(*pid);
+ if gateway_group_has_owned_member(leader) {
+ failed.push(leader.pid);
}
}
failed
@@ -812,14 +900,7 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result<Child> {
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
- #[cfg(unix)]
- unsafe {
- use std::os::unix::process::CommandExt;
- child_command.pre_exec(|| {
- libc::setsid();
- Ok(())
- });
- }
+ configure_process_group(&mut child_command);
let mut child = child_command.spawn()?;
if let Some(stdout) = child.stdout.take() {
let label = node.name();
@@ -832,30 +913,30 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result<Child> {
Ok(child)
}
-fn prefix_output(label: &str, reader: impl std::io::Read) {
- use std::io::{BufRead, BufReader};
- for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
- println!("[{label}] {line}");
- }
-}
-
pub fn stop_processes(processes: &mut [Child]) {
let mut children = processes.iter_mut().collect::<Vec<_>>();
- stop_child_processes(&mut children);
+ stop_child_processes_with_grace(&mut children, ROOT_STOP_GRACE_PERIOD);
+ stop_idle_epmd();
}
pub fn stop_child_processes(processes: &mut [&mut Child]) {
+ stop_child_processes_with_grace(processes, NESTED_STOP_GRACE_PERIOD);
+}
+
+fn stop_child_processes_with_grace(processes: &mut [&mut Child], grace_period: Duration) {
for process in processes.iter_mut() {
- if process.try_wait().ok().flatten().is_some() {
- continue;
+ if let Err(error) = terminate_process_group(process) {
+ eprintln!(
+ "Failed to terminate process group {}: {error}",
+ process.id()
+ );
}
- terminate_process(process);
}
- let deadline = Instant::now() + STOP_GRACE_PERIOD;
+ let deadline = Instant::now() + grace_period;
loop {
- let all_exited = processes
- .iter_mut()
- .all(|process| process.try_wait().ok().flatten().is_some());
+ let all_exited = processes.iter_mut().all(|process| {
+ process.try_wait().ok().flatten().is_some() && !process_group_running(process)
+ });
if all_exited {
return;
}
@@ -865,38 +946,30 @@ pub fn stop_child_processes(processes: &mut [&mut Child]) {
std::thread::sleep(Duration::from_millis(100));
}
for process in processes {
- if process.try_wait().ok().flatten().is_some() {
- continue;
+ if process.try_wait().ok().flatten().is_none() || process_group_running(process) {
+ force_kill_process_group(process);
}
- force_kill_process(process);
}
}
-#[cfg(unix)]
-fn terminate_process(process: &mut Child) {
- unsafe {
- libc::kill(-(process.id() as i32), libc::SIGTERM);
+fn stop_idle_epmd() {
+ let output = match Command::new("epmd").arg("-kill").output() {
+ Ok(output) => output,
+ Err(error) => {
+ eprintln!("Failed to stop Erlang port mapper: {error}");
+ return;
+ }
+ };
+ if output.status.success() {
+ return;
}
-}
-
-#[cfg(not(unix))]
-fn terminate_process(process: &mut Child) {
- let _ = process.kill();
-}
-
-#[cfg(unix)]
-fn force_kill_process(process: &mut Child) {
- unsafe {
- libc::kill(-(process.id() as i32), libc::SIGKILL);
+ let mut message = output.stdout;
+ message.extend(output.stderr);
+ let message = String::from_utf8_lossy(&message);
+ if message.contains("Cannot connect to local epmd") {
+ return;
}
- let _ = process.kill();
- let _ = process.wait();
-}
-
-#[cfg(not(unix))]
-fn force_kill_process(process: &mut Child) {
- let _ = process.kill();
- let _ = process.wait();
+ eprintln!("Failed to stop Erlang port mapper: {}", message.trim());
}
pub fn build_gateway_command(config_dir: &Path) -> Result<Vec<String>> {
diff --git a/tools/dev/src/gateway_reload.rs b/tools/dev/src/gateway_reload.rs
deleted file mode 100644
index ee9f694a3..000000000
--- a/tools/dev/src/gateway_reload.rs
+++ /dev/null
@@ -1,420 +0,0 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-use crate::gateway::{GatewayNode, gateway_dir};
-use anyhow::{Context, Result, bail};
-use std::collections::{BTreeMap, BTreeSet};
-use std::env;
-use std::fs;
-use std::io::Read;
-use std::path::{Path, PathBuf};
-use std::process::{Child, Command, Stdio};
-use std::sync::mpsc::{Receiver, channel};
-use std::time::{Duration, Instant, SystemTime};
-
-const WATCH_POLL_INTERVAL: Duration = Duration::from_millis(1000);
-const WATCH_SETTLE_INTERVAL: Duration = Duration::from_millis(300);
-const RELOAD_TIMEOUT: Duration = Duration::from_secs(60);
-const WATCHED_SOURCE_EXTENSIONS: &[&str] = &["erl", "hrl", "src", "rs", "toml", "config"];
-
-pub type FileState = BTreeMap<PathBuf, (SystemTime, u64)>;
-
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct ArtifactState {
- pub beams: FileState,
- pub nifs: FileState,
-}
-
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct ArtifactDiff {
- pub modules: Vec<String>,
- pub nifs_changed: bool,
-}
-
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct ReloadOutcome {
- pub failed_nodes: Vec<String>,
-}
-
-pub fn hot_reload_enabled() -> bool {
- env::var("FLUXER_DEV_GATEWAY_HOT_RELOAD")
- .map(|value| !matches!(value.to_ascii_lowercase().as_str(), "0" | "false" | "no"))
- .unwrap_or(true)
-}
-
-pub fn spawn_source_watcher() -> Receiver<()> {
- let (sender, receiver) = channel();
- std::thread::spawn(move || {
- let mut previous = scan_sources();
- loop {
- std::thread::sleep(WATCH_POLL_INTERVAL);
- let mut current = scan_sources();
- if current == previous {
- continue;
- }
- loop {
- std::thread::sleep(WATCH_SETTLE_INTERVAL);
- let settled = scan_sources();
- if settled == current {
- break;
- }
- current = settled;
- }
- previous = current;
- if sender.send(()).is_err() {
- return;
- }
- }
- });
- receiver
-}
-
-fn scan_sources() -> FileState {
- let dir = gateway_dir();
- let mut state = FileState::new();
- for root in [dir.join("src"), dir.join("include"), dir.join("native")] {
- scan_tree(&mut state, &root);
- }
- for path in [
- dir.join("rebar.config"),
- dir.join("rebar.config.script"),
- dir.join("rebar.lock"),
- ] {
- record_file(&mut state, &path);
- }
- state
-}
-
-fn scan_tree(state: &mut FileState, root: &Path) {
- let Ok(entries) = fs::read_dir(root) else {
- return;
- };
- for entry in entries.flatten() {
- let path = entry.path();
- let Ok(metadata) = entry.metadata() else {
- continue;
- };
- if metadata.is_dir() {
- if !is_skipped_dir(&path) {
- scan_tree(state, &path);
- }
- } else if is_watched_source(&path)
- && let Ok(modified) = metadata.modified()
- {
- state.insert(path, (modified, metadata.len()));
- }
- }
-}
-
-fn record_file(state: &mut FileState, path: &Path) {
- if let Ok(metadata) = fs::metadata(path)
- && let Ok(modified) = metadata.modified()
- {
- state.insert(path.to_path_buf(), (modified, metadata.len()));
- }
-}
-
-fn is_skipped_dir(path: &Path) -> bool {
- path.file_name()
- .and_then(|name| name.to_str())
- .is_some_and(|name| name == "target" || name.starts_with('.'))
-}
-
-fn is_watched_source(path: &Path) -> bool {
- path.extension()
- .and_then(|extension| extension.to_str())
- .is_some_and(|extension| WATCHED_SOURCE_EXTENSIONS.contains(&extension))
-}
-
-pub fn snapshot_artifacts() -> ArtifactState {
- let mut state = ArtifactState::default();
- let lib_root = gateway_dir().join("_build/default/lib");
- if let Ok(entries) = fs::read_dir(&lib_root) {
- for entry in entries.flatten() {
- scan_artifact_dir(&mut state.beams, &entry.path().join("ebin"), "beam");
- }
- }
- scan_artifact_dir(&mut state.nifs, &gateway_dir().join("priv"), "so");
- state
-}
-
-fn scan_artifact_dir(state: &mut FileState, dir: &Path, extension: &str) {
- let Ok(entries) = fs::read_dir(dir) else {
- return;
- };
- for entry in entries.flatten() {
- let path = entry.path();
- if path
- .extension()
- .is_some_and(|candidate| candidate == extension)
- && let Ok(metadata) = entry.metadata()
- && let Ok(modified) = metadata.modified()
- {
- state.insert(path, (modified, metadata.len()));
- }
- }
-}
-
-pub fn changed_artifacts(before: &ArtifactState, after: &ArtifactState) -> ArtifactDiff {
- let modules = after
- .beams
- .iter()
- .filter(|(path, state)| before.beams.get(*path) != Some(*state))
- .filter_map(|(path, _)| Some(path.file_stem()?.to_str()?.to_owned()))
- .collect::<BTreeSet<_>>()
- .into_iter()
- .collect();
- let nifs_changed = after
- .nifs
- .iter()
- .any(|(path, state)| before.nifs.get(path) != Some(state));
- ArtifactDiff {
- modules,
- nifs_changed,
- }
-}
-
-pub fn build_reload_eval(nodes: &[String], modules: &[String]) -> String {
- let node_list = nodes
- .iter()
- .map(|node| format!("'{node}'"))
- .collect::<Vec<_>>()
- .join(",");
- let module_list = modules
- .iter()
- .map(|module| format!("'{module}'"))
- .collect::<Vec<_>>()
- .join(",");
- format!(
- "Nodes = [{node_list}], \
- Mods = [{module_list}], \
- Failed = [Node || Node <- Nodes, \
- case net_adm:ping(Node) of \
- pong -> \
- Errors = [Mod || Mod <- Mods, \
- begin \
- rpc:call(Node, code, purge, [Mod], 10000), \
- case rpc:call(Node, code, load_file, [Mod], 10000) of \
- {{module, Mod}} -> false; \
- Other -> \
- io:format(\"gateway_reload_error ~s ~s ~0p~n\", [Node, Mod, Other]), \
- true \
- end \
- end], \
- Errors =/= []; \
- pang -> \
- io:format(\"gateway_reload_node_down ~s~n\", [Node]), \
- true \
- end], \
- [io:format(\"gateway_reload_failed_node ~s~n\", [Failed1]) || Failed1 <- Failed], \
- io:format(\"gateway_reload_done~n\"), \
- halt(0)."
- )
-}
-
-pub fn hot_reload_modules(
- nodes: &[GatewayNode],
- modules: &[String],
- cookie: &str,
-) -> Result<ReloadOutcome> {
- assert!(!modules.is_empty());
- let node_names = nodes
- .iter()
- .map(GatewayNode::erlang_name)
- .collect::<Vec<_>>();
- let eval = build_reload_eval(&node_names, modules);
- let reloader_name = format!("fluxer_dev_reload_{}@127.0.0.1", std::process::id());
- let mut child = Command::new("erl")
- .args([
- "-hidden",
- "-noshell",
- "-name",
- &reloader_name,
- "-setcookie",
- cookie,
- "-eval",
- &eval,
- ])
- .stdin(Stdio::null())
- .stdout(Stdio::piped())
- .stderr(Stdio::piped())
- .spawn()
- .context("failed to spawn gateway reload shell")?;
- let stdout = child.stdout.take().map(spawn_reader);
- let stderr = child.stderr.take().map(spawn_reader);
- let status = wait_with_deadline(&mut child, RELOAD_TIMEOUT)?;
- let mut output = String::new();
- for handle in [stdout, stderr].into_iter().flatten() {
- if let Ok(text) = handle.join() {
- output.push_str(&text);
- }
- }
- for line in output.lines().filter(|line| !line.trim().is_empty()) {
- println!("[gateway:reload] {line}");
- }
- if !status.success() {
- bail!(
- "gateway reload shell exited with status {}",
- status.code().unwrap_or(1)
- );
- }
- if !output.lines().any(|line| line == "gateway_reload_done") {
- bail!("gateway reload shell did not report completion");
- }
- Ok(parse_reload_outcome(&output))
-}
-
-pub fn parse_reload_outcome(output: &str) -> ReloadOutcome {
- let failed_nodes = output
- .lines()
- .filter_map(|line| line.strip_prefix("gateway_reload_failed_node "))
- .map(|node| node.trim().to_owned())
- .collect::<BTreeSet<_>>()
- .into_iter()
- .collect();
- ReloadOutcome { failed_nodes }
-}
-
-fn spawn_reader(stream: impl Read + Send + 'static) -> std::thread::JoinHandle<String> {
- std::thread::spawn(move || {
- let mut reader = stream;
- let mut text = String::new();
- let mut bytes = Vec::new();
- if reader.read_to_end(&mut bytes).is_ok() {
- text = String::from_utf8_lossy(&bytes).into_owned();
- }
- text
- })
-}
-
-fn wait_with_deadline(child: &mut Child, timeout: Duration) -> Result<std::process::ExitStatus> {
- let deadline = Instant::now() + timeout;
- loop {
- if let Some(status) = child.try_wait()? {
- return Ok(status);
- }
- if Instant::now() >= deadline {
- let _ = child.kill();
- let _ = child.wait();
- bail!(
- "gateway reload shell timed out after {}s",
- timeout.as_secs()
- );
- }
- std::thread::sleep(Duration::from_millis(100));
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn file_state(entries: &[(&str, u64, u64)]) -> FileState {
- entries
- .iter()
- .map(|(path, seconds, size)| {
- (
- PathBuf::from(path),
- (
- SystemTime::UNIX_EPOCH + Duration::from_secs(*seconds),
- *size,
- ),
- )
- })
- .collect()
- }
-
- #[test]
- fn changed_artifacts_detects_new_and_modified_beams() {
- let before = ArtifactState {
- beams: file_state(&[("ebin/a.beam", 1, 10), ("ebin/b.beam", 1, 20)]),
- nifs: FileState::new(),
- };
- let after = ArtifactState {
- beams: file_state(&[
- ("ebin/a.beam", 2, 10),
- ("ebin/b.beam", 1, 20),
- ("ebin/c.beam", 1, 30),
- ]),
- nifs: FileState::new(),
- };
- let diff = changed_artifacts(&before, &after);
- assert_eq!(diff.modules, vec!["a".to_owned(), "c".to_owned()]);
- assert!(!diff.nifs_changed);
- }
-
- #[test]
- fn changed_artifacts_ignores_unchanged_state() {
- let state = ArtifactState {
- beams: file_state(&[("ebin/a.beam", 1, 10)]),
- nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
- };
- let diff = changed_artifacts(&state, &state.clone());
- assert!(diff.modules.is_empty());
- assert!(!diff.nifs_changed);
- }
-
- #[test]
- fn changed_artifacts_flags_nif_changes() {
- let before = ArtifactState {
- beams: FileState::new(),
- nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
- };
- let after = ArtifactState {
- beams: FileState::new(),
- nifs: file_state(&[("priv/a_nif.so", 2, 11)]),
- };
- assert!(changed_artifacts(&before, &after).nifs_changed);
- }
-
- #[test]
- fn reload_eval_quotes_nodes_and_modules() {
- let eval = build_reload_eval(
- &["[email protected]".to_owned()],
- &["gateway_compress".to_owned(), "push".to_owned()],
- );
- assert!(eval.contains("Nodes = ['[email protected]']"));
- assert!(eval.contains("Mods = ['gateway_compress','push']"));
- assert!(eval.contains("halt(0)."));
- }
-
- #[test]
- fn reload_outcome_parses_failed_nodes() {
- let output = "gateway_reload_error n1 mod {error,nofile}\n\
- gateway_reload_failed_node [email protected]\n\
- gateway_reload_failed_node [email protected]\n\
- gateway_reload_done\n";
- let outcome = parse_reload_outcome(output);
- assert_eq!(
- outcome.failed_nodes,
- vec!["[email protected]".to_owned()]
- );
- }
-
- #[test]
- fn reload_outcome_empty_when_no_failures() {
- assert!(
- parse_reload_outcome("gateway_reload_done\n")
- .failed_nodes
- .is_empty()
- );
- }
-
- #[test]
- fn watched_source_filter_accepts_known_extensions() {
- assert!(is_watched_source(Path::new("src/push.erl")));
- assert!(is_watched_source(Path::new("include/gateway.hrl")));
- assert!(is_watched_source(Path::new("src/fluxer_gateway.app.src")));
- assert!(is_watched_source(Path::new("native/a_nif/src/lib.rs")));
- assert!(is_watched_source(Path::new("native/a_nif/Cargo.toml")));
- assert!(!is_watched_source(Path::new("ebin/push.beam")));
- assert!(!is_watched_source(Path::new("src/.push.erl.swp")));
- assert!(!is_watched_source(Path::new("native/a_nif/Cargo.lock")));
- }
-
- #[test]
- fn skipped_dirs_cover_build_output_and_hidden_dirs() {
- assert!(is_skipped_dir(Path::new("native/a_nif/target")));
- assert!(is_skipped_dir(Path::new("src/.git")));
- assert!(!is_skipped_dir(Path::new("src/gateway")));
- }
-}
diff --git a/tools/dev/src/lib.rs b/tools/dev/src/lib.rs
index 11f54c421..32122d28b 100644
--- a/tools/dev/src/lib.rs
+++ b/tools/dev/src/lib.rs
@@ -7,7 +7,6 @@ pub mod dev;
pub mod disclaim;
pub mod env;
pub mod gateway;
-pub mod gateway_reload;
pub mod manifest;
pub mod media_external;
pub mod media_proxy;
diff --git a/tools/dev/src/main.rs b/tools/dev/src/main.rs
index 822e6695a..4f3f0a2ca 100644
--- a/tools/dev/src/main.rs
+++ b/tools/dev/src/main.rs
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use anyhow::Result;
+use anyhow::{Context, Result, bail};
use clap::{Args, Parser, Subcommand};
use fluxer_dev::cassandra::{
apply_schema, compute_diff, render_target_schema, verify_schema, write_diff_file,
@@ -11,9 +11,19 @@ use fluxer_dev::desktop::{
};
use fluxer_dev::env::merge_default_env_with_current;
use fluxer_dev::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL};
-use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT_LOCAL_ENV_FILE};
+use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};
use std::collections::BTreeMap;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
+use std::process::Command as ProcessCommand;
+
+const DEV_INFRA_SERVICES: &[&str] = &[
+ "postgres",
+ "valkey",
+ "nats",
+ "livekit",
+ "meilisearch",
+ "mailpit",
+];
#[derive(Debug, Parser)]
#[command(name = "fluxer-dev")]
@@ -34,6 +44,7 @@ enum Command {
Proxy(ProxyArgs),
Dev(DevArgs),
RustServices(RustServicesArgs),
+ Infra(InfraArgs),
Cassandra(CassandraArgs),
Desktop(DesktopArgs),
MediaProxy(MediaProxyArgs),
@@ -44,8 +55,6 @@ enum Command {
struct BootstrapArgs {
#[arg(long)]
skip_install: bool,
- #[arg(long)]
- skip_desktop_install: bool,
}
#[derive(Debug, Args)]
@@ -76,6 +85,19 @@ struct RustServicesArgs {
services: Vec<String>,
}
+#[derive(Debug, Args)]
+struct InfraArgs {
+ #[command(subcommand)]
+ command: InfraCommand,
+}
+
+#[derive(Debug, Subcommand)]
+enum InfraCommand {
+ Start,
+ Stop,
+ Status,
+}
+
#[derive(Debug, Args)]
struct CassandraArgs {
#[command(subcommand)]
@@ -193,10 +215,12 @@ async fn main() -> Result<()> {
match cli.command {
Command::Bootstrap(args) => {
- fluxer_dev::bootstrap::bootstrap(args.skip_install, args.skip_desktop_install).await?;
+ fluxer_dev::bootstrap::bootstrap(args.skip_install).await?;
}
Command::PostStart => fluxer_dev::bootstrap::post_start().await?,
- Command::Gateway(args) if args.mode == "single" => fluxer_dev::gateway::run_gateway()?,
+ Command::Gateway(args) if args.mode == "single" => {
+ std::process::exit(fluxer_dev::gateway::run_gateway().await?)
+ }
Command::Gateway(_) => {
std::process::exit(fluxer_dev::gateway::run_gateway_cluster().await?)
}
@@ -216,6 +240,7 @@ async fn main() -> Result<()> {
Command::RustServices(args) => {
std::process::exit(fluxer_dev::rust_services::run_rust_services(&args.services).await?)
}
+ Command::Infra(args) => run_infra(args.command)?,
Command::Cassandra(args) => match args.command {
CassandraCommand::Diff { output } => {
let diff = compute_diff(None).await?;
@@ -315,3 +340,78 @@ fn apply_default_env() -> Result<()> {
}
Ok(())
}
+
+fn run_infra(command: InfraCommand) -> Result<()> {
+ let project = compose_project_name()?;
+ let compose_file = ROOT.join(".devcontainer/docker-compose.yml");
+ let mut args = vec![
+ "compose".to_owned(),
+ "--project-name".to_owned(),
+ project,
+ "-f".to_owned(),
+ compose_file.display().to_string(),
+ ];
+ match command {
+ InfraCommand::Start => args.push("start".to_owned()),
+ InfraCommand::Stop => args.push("stop".to_owned()),
+ InfraCommand::Status => {
+ args.push("ps".to_owned());
+ args.push("--all".to_owned());
+ }
+ }
+ args.extend(
+ DEV_INFRA_SERVICES
+ .iter()
+ .map(|service| (*service).to_owned()),
+ );
+ let status = ProcessCommand::new("docker")
+ .args(&args)
+ .status()
+ .context("failed to run Docker Compose for the dev infrastructure")?;
+ if !status.success() {
+ bail!("Docker Compose dev infrastructure command failed with {status}");
+ }
+ Ok(())
+}
+
+fn compose_project_name() -> Result<String> {
+ if Path::new("/.dockerenv").exists() {
+ let container = std::fs::read_to_string("/etc/hostname")
+ .context("failed to read the current devcontainer hostname")?;
+ let container = container.trim();
+ if container.is_empty()
+ || !container
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-'))
+ {
+ bail!("Current devcontainer hostname is invalid");
+ }
+ let output = ProcessCommand::new("docker")
+ .args([
+ "inspect",
+ "--format",
+ "{{ index .Config.Labels \"com.docker.compose.project\" }}",
+ container,
+ ])
+ .output()
+ .context("failed to inspect the current devcontainer Compose project")?;
+ if !output.status.success() {
+ bail!(
+ "Could not discover the current devcontainer Compose project: {}",
+ String::from_utf8_lossy(&output.stderr).trim()
+ );
+ }
+ let project = String::from_utf8(output.stdout)
+ .context("devcontainer Compose project label is not valid UTF-8")?
+ .trim()
+ .to_owned();
+ if project.is_empty() || project == "<no value>" {
+ bail!("Current container has no com.docker.compose.project label");
+ }
+ return Ok(project);
+ }
+ std::env::var("COMPOSE_PROJECT_NAME")
+ .ok()
+ .filter(|project| !project.trim().is_empty())
+ .context("COMPOSE_PROJECT_NAME must be set when managing dev infrastructure from the host")
+}
diff --git a/tools/dev/src/manifest.rs b/tools/dev/src/manifest.rs
index 99d0a613c..7dbae7836 100644
--- a/tools/dev/src/manifest.rs
+++ b/tools/dev/src/manifest.rs
@@ -7,6 +7,7 @@ pub const LOOPBACK_HOST: &str = "127.0.0.1";
pub const ANY_HOST: &str = "0.0.0.0";
pub const DEV_PROXY_PORT: u16 = 8088;
+pub const DEV_PROXY_GATEWAY_PORTS_ENV: &str = "FLUXER_DEV_PROXY_GATEWAY_PORTS";
pub const APP_PORT: u16 = 3000;
pub const APP_PROXY_PORT: u16 = 8773;
pub const ADMIN_PORT: u16 = 3020;
diff --git a/tools/dev/src/media_proxy.rs b/tools/dev/src/media_proxy.rs
index 414303011..e8dc3c50f 100644
--- a/tools/dev/src/media_proxy.rs
+++ b/tools/dev/src/media_proxy.rs
@@ -17,6 +17,8 @@ const DEV_S3_ACCESS_KEY_ID: &str = "fluxer";
const DEV_S3_SECRET_ACCESS_KEY: &str = "fluxer-secret";
const DEV_S3_HOST: &str = "127.0.0.1";
const DEV_S3_PORT: u16 = 8333;
+const DEV_SEAWEEDFS_STOP_TIMEOUT: Duration = Duration::from_secs(15);
+const DEV_SEAWEEDFS_KILL_TIMEOUT: Duration = Duration::from_secs(2);
pub async fn run_dev_media_doctor(
repair: bool,
@@ -35,10 +37,20 @@ pub async fn run_dev_media_doctor(
}
pub async fn ensure_dev_object_store(repair: bool, repair_timeout_secs: u64) -> Result<()> {
+ let managed_process_running =
+ read_dev_seaweedfs_pid()?.is_some_and(managed_seaweedfs_process_running);
+ if repair && !tcp_reachable(DEV_S3_HOST, DEV_S3_PORT) && !managed_process_running {
+ println!("SeaweedFS S3 is not running; starting the managed development instance.");
+ start_dev_seaweedfs()?;
+ wait_s3_api(repair_timeout_secs).await?;
+ ensure_s3_buckets()?;
+ return Ok(());
+ }
+
match wait_s3_api(5).await {
Ok(()) => {}
- Err(error) if repair => {
- println!("SeaweedFS S3 check failed: {error}");
+ Err(_) if repair => {
+ println!("SeaweedFS S3 is unresponsive; restarting the managed development instance.");
start_dev_seaweedfs()?;
wait_s3_api(repair_timeout_secs).await?;
}
@@ -128,7 +140,7 @@ fn start_dev_seaweedfs() -> Result<()> {
if let Some(pid) = read_dev_seaweedfs_pid()? {
if managed_seaweedfs_process_running(pid) {
eprintln!("Stopping unresponsive managed SeaweedFS process {pid}.");
- stop_managed_seaweedfs_process(pid);
+ stop_managed_seaweedfs_process(pid)?;
}
let _ = fs::remove_file(DEV_SEAWEEDFS_PID_FILE.as_path());
}
@@ -141,15 +153,24 @@ fn start_dev_seaweedfs() -> Result<()> {
let log_path = DEV_LOG_DIR.join("seaweedfs.log");
let log = OpenOptions::new()
.create(true)
- .append(true)
+ .write(true)
+ .truncate(true)
.open(&log_path)
.with_context(|| format!("failed to open SeaweedFS log {}", log_path.display()))?;
let stderr = log
.try_clone()
.context("failed to clone SeaweedFS log handle")?;
let data_dir_arg = format!("-dir={}", DEV_SEAWEEDFS_DIR.display());
- let child = Command::new(weed)
- .args(["-logtostderr=true", "mini", &data_dir_arg])
+ let mut child = Command::new(weed)
+ .args([
+ "-logtostderr=true",
+ "mini",
+ &data_dir_arg,
+ "-admin.ui=false",
+ "-filer.disableDirListing",
+ "-s3.port.iceberg=0",
+ "-webdav=false",
+ ])
.env("AWS_ACCESS_KEY_ID", DEV_S3_ACCESS_KEY_ID)
.env("AWS_SECRET_ACCESS_KEY", DEV_S3_SECRET_ACCESS_KEY)
.env("S3_BUCKET", DEV_S3_BUCKETS)
@@ -159,12 +180,16 @@ fn start_dev_seaweedfs() -> Result<()> {
.spawn()
.context("failed to start SeaweedFS")?;
let pid = child.id();
- fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()).with_context(|| {
- format!(
- "failed to write SeaweedFS pid file {}",
- DEV_SEAWEEDFS_PID_FILE.display()
- )
- })?;
+ if let Err(error) = fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()) {
+ let _ = child.kill();
+ let _ = child.wait();
+ return Err(error).with_context(|| {
+ format!(
+ "failed to write SeaweedFS pid file {}",
+ DEV_SEAWEEDFS_PID_FILE.display()
+ )
+ });
+ }
println!(
"Started SeaweedFS dev object store with pid {pid}; logs: {}",
log_path.display()
@@ -195,20 +220,44 @@ fn managed_seaweedfs_process_running(pid: u32) -> bool {
}
#[cfg(unix)]
-fn stop_managed_seaweedfs_process(pid: u32) {
- unsafe {
- libc::kill(pid as i32, libc::SIGTERM);
+fn stop_managed_seaweedfs_process(pid: u32) -> Result<()> {
+ let result = unsafe { libc::kill(pid as i32, libc::SIGTERM) };
+ if result == -1 {
+ let error = std::io::Error::last_os_error();
+ if error.raw_os_error() != Some(libc::ESRCH) {
+ return Err(error).context("failed to terminate managed SeaweedFS process");
+ }
}
- std::thread::sleep(Duration::from_secs(2));
- if managed_seaweedfs_process_running(pid) {
- unsafe {
- libc::kill(pid as i32, libc::SIGKILL);
+
+ let terminate_deadline = Instant::now() + DEV_SEAWEEDFS_STOP_TIMEOUT;
+ while managed_seaweedfs_process_running(pid) && Instant::now() < terminate_deadline {
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ if !managed_seaweedfs_process_running(pid) {
+ return Ok(());
+ }
+
+ let result = unsafe { libc::kill(pid as i32, libc::SIGKILL) };
+ if result == -1 {
+ let error = std::io::Error::last_os_error();
+ if error.raw_os_error() != Some(libc::ESRCH) {
+ return Err(error).context("failed to kill unresponsive managed SeaweedFS process");
}
}
+ let kill_deadline = Instant::now() + DEV_SEAWEEDFS_KILL_TIMEOUT;
+ while managed_seaweedfs_process_running(pid) && Instant::now() < kill_deadline {
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ if managed_seaweedfs_process_running(pid) {
+ bail!("managed SeaweedFS process {pid} survived SIGKILL");
+ }
+ Ok(())
}
#[cfg(not(unix))]
-fn stop_managed_seaweedfs_process(_pid: u32) {}
+fn stop_managed_seaweedfs_process(_pid: u32) -> Result<()> {
+ Ok(())
+}
async fn check_dev_media_path(base_url: &str, path: &str) -> Result<()> {
let path = if path.starts_with('/') {
diff --git a/tools/dev/src/object_store.rs b/tools/dev/src/object_store.rs
index 4c9ee06a9..69bc4810e 100644
--- a/tools/dev/src/object_store.rs
+++ b/tools/dev/src/object_store.rs
@@ -168,11 +168,11 @@ fn s3_bucket_already_exists_output(output: &str) -> bool {
output.contains("BucketAlreadyExists") || output.contains("BucketAlreadyOwnedByYou")
}
-pub async fn bootstrap_schema_and_object_store() -> Result<()> {
+pub async fn bootstrap_schema() -> Result<()> {
if cassandra_backend_enabled() {
apply_schema(Some(config_from_env()?)).await?;
}
- ensure_s3_buckets()
+ Ok(())
}
fn cassandra_backend_enabled() -> bool {
diff --git a/tools/dev/src/proc.rs b/tools/dev/src/proc.rs
index 95282721e..6c5b35d00 100644
--- a/tools/dev/src/proc.rs
+++ b/tools/dev/src/proc.rs
@@ -4,9 +4,11 @@ use crate::env::merge_default_env_with_current;
use crate::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};
use anyhow::{Context, Result, bail};
use std::collections::{BTreeMap, VecDeque};
+use std::future::Future;
+use std::io::{BufRead, BufReader, Read};
use std::net::{TcpStream, ToSocketAddrs};
use std::path::Path;
-use std::process::{Command, Output, Stdio};
+use std::process::{Child, Command, Output, Stdio};
use std::time::{Duration, Instant};
use tokio::time::sleep;
@@ -172,6 +174,152 @@ pub fn run_command(args: &[&str], options: RunOptions<'_>) -> Result<Output> {
Ok(output)
}
+pub async fn run_command_interruptible(
+ args: &[&str],
+ options: RunOptions<'_>,
+ shutdown: &mut ShutdownSignal,
+) -> Result<AwaitOutcome<Output>> {
+ if options.capture {
+ bail!("interruptible commands do not support captured output");
+ }
+ println!("$ {}", format_command(args));
+ let env = merged_env(Some(&options.env), options.load_default_env)?;
+ let mut command = Command::new(args[0]);
+ command
+ .args(&args[1..])
+ .current_dir(options.cwd)
+ .env_clear()
+ .envs(env)
+ .stdin(Stdio::inherit())
+ .stdout(Stdio::inherit())
+ .stderr(Stdio::inherit());
+ configure_process_group(&mut command);
+ let mut child = command
+ .spawn()
+ .with_context(|| format!("failed to run {}", format_command(args)))?;
+ loop {
+ let status = match child.try_wait() {
+ Ok(status) => status,
+ Err(error) => {
+ stop_process_group(&mut child, Duration::from_secs(5)).await;
+ return Err(error)
+ .with_context(|| format!("failed to wait for {}", format_command(args)));
+ }
+ };
+ if let Some(status) = status {
+ stop_process_group(&mut child, Duration::from_secs(5)).await;
+ let output = Output {
+ status,
+ stdout: Vec::new(),
+ stderr: Vec::new(),
+ };
+ if options.check && !output.status.success() {
+ let code = output.status.code().unwrap_or(-1);
+ bail!(
+ "Command failed with exit code {code}: {}",
+ format_command(args)
+ );
+ }
+ return Ok(AwaitOutcome::Completed(output));
+ }
+ tokio::select! {
+ signal = shutdown.recv() => {
+ stop_process_group(&mut child, Duration::from_secs(5)).await;
+ return Ok(AwaitOutcome::Shutdown(signal));
+ }
+ _ = sleep(Duration::from_millis(100)) => {}
+ }
+ }
+}
+
+pub fn configure_process_group(command: &mut Command) {
+ #[cfg(unix)]
+ unsafe {
+ use std::os::unix::process::CommandExt;
+ command.pre_exec(|| {
+ if libc::setsid() == -1 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ });
+ }
+}
+
+#[cfg(unix)]
+pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> {
+ signal_process_group(process, libc::SIGTERM)
+}
+
+#[cfg(not(unix))]
+pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> {
+ process.kill()
+}
+
+#[cfg(unix)]
+pub fn force_kill_process_group(process: &mut Child) {
+ if let Err(error) = signal_process_group(process, libc::SIGKILL) {
+ eprintln!(
+ "Failed to send SIGKILL to process group {}: {error}",
+ process.id()
+ );
+ }
+ let _ = process.kill();
+ let _ = process.wait();
+}
+
+#[cfg(unix)]
+pub fn process_group_running(process: &mut Child) -> bool {
+ if unsafe { libc::kill(-(process.id() as i32), 0) } == 0 {
+ return true;
+ }
+ let error = std::io::Error::last_os_error();
+ error.raw_os_error() != Some(libc::ESRCH)
+}
+
+#[cfg(not(unix))]
+pub fn process_group_running(process: &mut Child) -> bool {
+ process.try_wait().ok().flatten().is_none()
+}
+
+#[cfg(not(unix))]
+pub fn force_kill_process_group(process: &mut Child) {
+ let _ = process.kill();
+ let _ = process.wait();
+}
+
+#[cfg(unix)]
+fn signal_process_group(process: &Child, signal: libc::c_int) -> std::io::Result<()> {
+ if unsafe { libc::kill(-(process.id() as i32), signal) } == 0 {
+ return Ok(());
+ }
+ let error = std::io::Error::last_os_error();
+ if error.raw_os_error() == Some(libc::ESRCH) {
+ return Ok(());
+ }
+ Err(error)
+}
+
+pub async fn stop_process_group(process: &mut Child, grace_period: Duration) {
+ if let Err(error) = terminate_process_group(process) {
+ eprintln!(
+ "Failed to terminate process group {}: {error}",
+ process.id()
+ );
+ }
+ let deadline = Instant::now() + grace_period;
+ loop {
+ let leader_exited = process.try_wait().ok().flatten().is_some();
+ if leader_exited && !process_group_running(process) {
+ return;
+ }
+ if Instant::now() >= deadline {
+ force_kill_process_group(process);
+ return;
+ }
+ sleep(Duration::from_millis(100)).await;
+ }
+}
+
pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> Result<()> {
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_error = None;
@@ -199,6 +347,19 @@ pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> R
}
pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
+ wait_http_status(name, url, timeout_secs, |status| status.as_u16() < 500).await
+}
+
+pub async fn wait_http_success(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
+ wait_http_status(name, url, timeout_secs, |status| status.is_success()).await
+}
+
+async fn wait_http_status(
+ name: &str,
+ url: &str,
+ timeout_secs: u64,
+ accepts: impl Fn(reqwest::StatusCode) -> bool,
+) -> Result<()> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
@@ -206,7 +367,7 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
let mut last_error = None;
while Instant::now() < deadline {
match client.get(url).send().await {
- Ok(response) if response.status().as_u16() < 500 => {
+ Ok(response) if accepts(response.status()) => {
println!("{name} is reachable at {url}");
return Ok(());
}
@@ -221,6 +382,30 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
);
}
+pub fn prefix_output(label: &str, reader: impl Read) {
+ let mut reader = BufReader::new(reader);
+ let mut line = Vec::new();
+ loop {
+ line.clear();
+ match reader.read_until(b'\n', &mut line) {
+ Ok(0) => return,
+ Ok(_) => {
+ if line.last() == Some(&b'\n') {
+ line.pop();
+ }
+ if line.last() == Some(&b'\r') {
+ line.pop();
+ }
+ println!("[{label}] {}", String::from_utf8_lossy(&line));
+ }
+ Err(error) => {
+ eprintln!("[{label}] output reader failed: {error}");
+ return;
+ }
+ }
+ }
+}
+
pub const RESTART_WINDOW: Duration = Duration::from_secs(60);
pub const RESTART_LIMIT: usize = 5;
@@ -262,6 +447,29 @@ impl ShutdownSignal {
}
}
+pub enum AwaitOutcome<T> {
+ Completed(T),
+ Shutdown(&'static str),
+}
+
+pub async fn await_or_shutdown<T, F>(shutdown: &mut ShutdownSignal, future: F) -> AwaitOutcome<T>
+where
+ F: Future<Output = T>,
+{
+ tokio::select! {
+ biased;
+ signal = shutdown.recv() => AwaitOutcome::Shutdown(signal),
+ output = future => AwaitOutcome::Completed(output),
+ }
+}
+
+pub async fn pending_shutdown(shutdown: &mut ShutdownSignal) -> Option<&'static str> {
+ match await_or_shutdown(shutdown, std::future::ready(())).await {
+ AwaitOutcome::Completed(()) => None,
+ AwaitOutcome::Shutdown(signal) => Some(signal),
+ }
+}
+
#[cfg(not(unix))]
pub struct ShutdownSignal;
diff --git a/tools/dev/src/proxy.rs b/tools/dev/src/proxy.rs
index 0920f4c11..5f88001ee 100644
--- a/tools/dev/src/proxy.rs
+++ b/tools/dev/src/proxy.rs
@@ -1,6 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use crate::manifest::{ANY_HOST, DEV_PROXY_PORT, LOCAL_APP_URL, PROXY_ROUTES, ProxyRoute};
+use crate::manifest::{
+ ANY_HOST, DEV_PROXY_GATEWAY_PORTS_ENV, DEV_PROXY_PORT, GATEWAY_PORT, LOCAL_APP_URL,
+ PROXY_ROUTES, ProxyRoute,
+};
use anyhow::{Context, Result, bail};
use axum::{
Router,
@@ -14,7 +17,7 @@ use hyper_util::rt::TokioIo;
use std::collections::HashMap;
use std::env;
use std::net::SocketAddr;
-use std::sync::{LazyLock, Mutex};
+use std::sync::{Arc, LazyLock, Mutex};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
@@ -26,6 +29,7 @@ static ROUTE_CURSORS: LazyLock<Mutex<HashMap<&'static str, usize>>> =
#[derive(Clone)]
struct ProxyState {
http_client: reqwest::Client,
+ gateway_ports: Arc<[u16]>,
}
const BLOCKED_REQUEST_HEADERS: &[&str] = &[
@@ -67,9 +71,21 @@ pub async fn run_proxy(host: &str, port: u16) -> Result<()> {
.tcp_nodelay(true)
.build()
.context("failed to build dev proxy HTTP client")?;
+ let gateway_ports = gateway_proxy_ports()?;
+ println!(
+ "Fluxer dev proxy gateway ports: {}",
+ gateway_ports
+ .iter()
+ .map(u16::to_string)
+ .collect::<Vec<_>>()
+ .join(",")
+ );
let app = Router::new()
.fallback(any(proxy_request))
- .with_state(ProxyState { http_client });
+ .with_state(ProxyState {
+ http_client,
+ gateway_ports,
+ });
axum::serve(
listener,
@@ -91,11 +107,29 @@ async fn proxy_request(
}
let route = route_for_path(&request_head.path);
+ let (target_host, target_port) = target_for_route(route, &state.gateway_ports);
if is_upgrade_request(&request_head) {
- return proxy_upgrade(request, request_head, route, client_addr).await;
+ return proxy_upgrade(
+ request,
+ request_head,
+ route,
+ target_host,
+ target_port,
+ client_addr,
+ )
+ .await;
}
- proxy_http(state, request, request_head, route, client_addr).await
+ proxy_http(
+ state,
+ request,
+ request_head,
+ route,
+ target_host,
+ target_port,
+ client_addr,
+ )
+ .await
}
async fn proxy_http(
@@ -103,9 +137,10 @@ async fn proxy_http(
request: Request<Body>,
request_head: RequestHead,
route: &'static ProxyRoute,
+ target_host: &'static str,
+ target_port: u16,
client_addr: SocketAddr,
) -> Response<Body> {
- let (target_host, target_port) = target_for_route(route);
let target_url = upstream_http_url(&request_head.path, route, target_host, target_port);
let (parts, body) = request.into_parts();
let method = parts.method.clone();
@@ -155,10 +190,11 @@ async fn proxy_upgrade(
mut request: Request<Body>,
request_head: RequestHead,
route: &'static ProxyRoute,
+ target_host: &'static str,
+ target_port: u16,
client_addr: SocketAddr,
) -> Response<Body> {
let on_upgrade = hyper::upgrade::on(&mut request);
- let (target_host, target_port) = target_for_route(route);
let mut target = match TcpStream::connect((target_host, target_port)).await {
Ok(target) => target,
Err(error) => return bad_gateway_response(error),
@@ -497,17 +533,59 @@ pub fn route_for_path(path: &str) -> &'static ProxyRoute {
PROXY_ROUTES.last().expect("proxy has fallback route")
}
-pub fn target_for_route(route: &'static ProxyRoute) -> (&'static str, u16) {
- if route.alternate_ports.is_empty() {
+pub fn target_for_route(route: &'static ProxyRoute, gateway_ports: &[u16]) -> (&'static str, u16) {
+ let uses_gateway_ports = route.prefix == "/gateway";
+ let port_count = if uses_gateway_ports {
+ gateway_ports.len()
+ } else {
+ route.alternate_ports.len() + 1
+ };
+ assert!(
+ port_count > 0,
+ "proxy route must have at least one target port"
+ );
+ if port_count == 1 {
+ if uses_gateway_ports {
+ return (route.host, gateway_ports[0]);
+ }
return (route.host, route.port);
}
- let ports = std::iter::once(route.port)
- .chain(route.alternate_ports.iter().copied())
- .collect::<Vec<_>>();
let mut cursors = ROUTE_CURSORS.lock().expect("route cursor lock poisoned");
let cursor = *cursors.get(route.prefix).unwrap_or(&0);
- cursors.insert(route.prefix, (cursor + 1) % ports.len());
- (route.host, ports[cursor])
+ cursors.insert(route.prefix, (cursor + 1) % port_count);
+ let port = if uses_gateway_ports {
+ gateway_ports[cursor]
+ } else if cursor == 0 {
+ route.port
+ } else {
+ route.alternate_ports[cursor - 1]
+ };
+ (route.host, port)
+}
+
+fn gateway_proxy_ports() -> Result<Arc<[u16]>> {
+ let raw = env::var(DEV_PROXY_GATEWAY_PORTS_ENV).unwrap_or_else(|_| GATEWAY_PORT.to_string());
+ let mut ports = Vec::new();
+ for token in raw.split(',') {
+ let token = token.trim();
+ if token.is_empty() {
+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains an empty port");
+ }
+ let port = token
+ .parse::<u16>()
+ .with_context(|| format!("Invalid port {token:?} in {DEV_PROXY_GATEWAY_PORTS_ENV}"))?;
+ if port == 0 {
+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} ports must be greater than zero");
+ }
+ if ports.contains(&port) {
+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains duplicate port {port}");
+ }
+ ports.push(port);
+ }
+ if ports.len() > 3 {
+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} supports at most three ports");
+ }
+ Ok(ports.into())
}
pub fn rewrite_path(path: &str, route: &ProxyRoute) -> String {
diff --git a/tools/dev/src/rust_services.rs b/tools/dev/src/rust_services.rs
index e32bcc00d..33b1c3099 100644
--- a/tools/dev/src/rust_services.rs
+++ b/tools/dev/src/rust_services.rs
@@ -1,10 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::manifest::{DEV_PROXY_PORT, MEDIA_PROXY_PORT, RustServiceSpec, rust_services};
-use crate::paths::ROOT;
+use crate::paths::{ROOT, TARGET_DIR};
use crate::proc::{
- RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,
- restart_budget_exceeded, run_command,
+ AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown,
+ configure_process_group, format_command, merged_env, pending_shutdown, prefix_output,
+ restart_budget_exceeded, run_command_interruptible,
};
use anyhow::{Result, bail};
use std::collections::{BTreeSet, VecDeque};
@@ -12,6 +13,8 @@ use std::env;
#[cfg(target_os = "linux")]
use std::fs;
use std::net::{SocketAddr, TcpListener};
+#[cfg(target_os = "linux")]
+use std::path::Path;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
use tokio::time::sleep;
@@ -34,26 +37,51 @@ impl SupervisedService {
pub async fn run_rust_services(service_names: &[String]) -> Result<i32> {
let selected = select_services(service_names)?;
+ let mut shutdown = ShutdownSignal::new()?;
cleanup_orphaned_service_processes(&selected).await?;
+ if let Some(signal) = pending_shutdown(&mut shutdown).await {
+ println!("Received {signal}; stopping Rust service startup...");
+ return Ok(0);
+ }
wait_for_service_ports_available(&selected)?;
- build_services(&selected)?;
- let mut shutdown = ShutdownSignal::new()?;
+ match build_services(&selected, &mut shutdown).await? {
+ AwaitOutcome::Completed(_) => {}
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping Rust service startup...");
+ return Ok(0);
+ }
+ }
let mut supervised = Vec::new();
for spec in &selected {
for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {
+ let child = match start_service(spec, mode, port) {
+ Ok(child) => child,
+ Err(error) => {
+ stop_supervised_services(&mut supervised);
+ return Err(error);
+ }
+ };
supervised.push(SupervisedService {
spec: spec.clone(),
mode,
port,
- child: start_service(spec, mode, port)?,
+ child,
restarts: VecDeque::new(),
});
}
}
loop {
- if let Err(error) = restart_exited_services(&mut supervised).await {
- stop_supervised_services(&mut supervised);
- return Err(error);
+ match await_or_shutdown(&mut shutdown, restart_exited_services(&mut supervised)).await {
+ AwaitOutcome::Completed(Ok(())) => {}
+ AwaitOutcome::Completed(Err(error)) => {
+ stop_supervised_services(&mut supervised);
+ return Err(error);
+ }
+ AwaitOutcome::Shutdown(signal) => {
+ println!("Received {signal}; stopping Rust service tasks...");
+ stop_supervised_services(&mut supervised);
+ return Ok(0);
+ }
}
tokio::select! {
signal = shutdown.recv() => {
@@ -71,6 +99,7 @@ async fn restart_exited_services(supervised: &mut [SupervisedService]) -> Result
let Some(status) = entry.child.try_wait()? else {
continue;
};
+ crate::gateway::stop_child_processes(&mut [&mut entry.child]);
if restart_budget_exceeded(&mut entry.restarts, Instant::now()) {
bail!(
"Rust service {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",
@@ -145,7 +174,10 @@ pub fn select_services(service_names: &[String]) -> Result<Vec<RustServiceSpec>>
.collect())
}
-fn build_services(services: &[RustServiceSpec]) -> Result<()> {
+async fn build_services(
+ services: &[RustServiceSpec],
+ shutdown: &mut ShutdownSignal,
+) -> Result<AwaitOutcome<std::process::Output>> {
let mut packages = Vec::new();
let mut seen = BTreeSet::new();
for spec in services {
@@ -156,7 +188,23 @@ fn build_services(services: &[RustServiceSpec]) -> Result<()> {
let mut args = vec!["cargo".to_owned(), "build".to_owned()];
args.extend(packages);
let refs = args.iter().map(String::as_str).collect::<Vec<_>>();
- run_command(&refs, RunOptions::default()).map(drop)
+ let env = if env::var_os("CARGO_BUILD_JOBS").is_none() {
+ vec![(
+ "CARGO_BUILD_JOBS".to_owned(),
+ Some(env::var("FLUXER_DEV_CARGO_JOBS").unwrap_or_else(|_| "2".to_owned())),
+ )]
+ } else {
+ Vec::new()
+ };
+ run_command_interruptible(
+ &refs,
+ RunOptions {
+ env,
+ ..RunOptions::default()
+ },
+ shutdown,
+ )
+ .await
}
fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child> {
@@ -172,14 +220,7 @@ fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child>
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
- #[cfg(unix)]
- unsafe {
- use std::os::unix::process::CommandExt;
- command.pre_exec(|| {
- libc::setsid();
- Ok(())
- });
- }
+ configure_process_group(&mut command);
let mut child = command.spawn()?;
if let Some(stdout) = child.stdout.take() {
let stdout_label = label.clone();
@@ -212,11 +253,20 @@ pub fn service_command(spec: &RustServiceSpec) -> Vec<String> {
format!("run -p {}", spec.package),
];
}
+ let target_dir = env::var_os("CARGO_TARGET_DIR")
+ .map(std::path::PathBuf::from)
+ .unwrap_or_else(|| TARGET_DIR.clone());
+ let target_dir = if target_dir.is_absolute() {
+ target_dir
+ } else {
+ ROOT.join(target_dir)
+ };
vec![
- "cargo".to_owned(),
- "run".to_owned(),
- "-p".to_owned(),
- spec.package.to_owned(),
+ target_dir
+ .join("debug")
+ .join(format!("{}{}", spec.package, std::env::consts::EXE_SUFFIX))
+ .display()
+ .to_string(),
]
}
@@ -238,18 +288,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String
.unwrap_or_else(|_| "nats://nats:4222".to_owned()),
),
),
- (
- "FLUXER_CASSANDRA_HOSTS".to_owned(),
- Some(env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned())),
- ),
- (
- "FLUXER_CASSANDRA_KEYSPACE".to_owned(),
- Some(env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned())),
- ),
- (
- "FLUXER_CASSANDRA_PORT".to_owned(),
- Some(env::var("FLUXER_CASSANDRA_PORT").unwrap_or_else(|_| "9042".to_owned())),
- ),
];
if mode == "shard" {
envs.push(("FLUXER_SVC_SHARD_ID".to_owned(), Some("0".to_owned())));
@@ -284,13 +322,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String
envs
}
-fn prefix_output(label: &str, reader: impl std::io::Read) {
- use std::io::{BufRead, BufReader};
- for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
- println!("[{label}] {line}");
- }
-}
-
fn wait_for_service_ports_available(services: &[RustServiceSpec]) -> Result<()> {
let conflicts = services
.iter()
@@ -333,7 +364,7 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Res
"Stopping orphaned Rust service process group(s): {}",
format_pids(&pids)
);
- let term_failed = signal_process_groups(&pids, libc::SIGTERM);
+ let term_failed = signal_process_groups(&leaders, libc::SIGTERM);
if !term_failed.is_empty() {
println!(
"SIGTERM delivery failed for orphaned Rust service pid(s): {}",
@@ -348,7 +379,12 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Res
return Ok(());
}
if Instant::now() >= deadline {
- let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);
+ let remaining_leaders = leaders
+ .iter()
+ .filter(|leader| remaining.contains(&leader.pid))
+ .cloned()
+ .collect::<Vec<_>>();
+ let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL);
sleep(Duration::from_millis(200)).await;
let survivors = surviving_service_group_pids(&leaders)?;
if survivors.is_empty() {
@@ -380,19 +416,85 @@ async fn cleanup_orphaned_service_processes(_services: &[RustServiceSpec]) -> Re
}
#[cfg(target_os = "linux")]
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq)]
struct RustServiceLeader {
+ pid: i32,
+ members: Vec<RustServiceProcessIdentity>,
+}
+
+#[cfg(target_os = "linux")]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct RustServiceProcessIdentity {
pid: i32,
starttime: u64,
}
+#[cfg(target_os = "linux")]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct RustServiceProcess {
+ pid: i32,
+ ppid: i32,
+ pgid: i32,
+ state: char,
+ starttime: u64,
+}
+
#[cfg(target_os = "linux")]
fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServiceLeader>> {
- let binaries = services
+ let current_exe = std::env::current_exe()?;
+ let processes = service_process_snapshot()?;
+ let mut owned_pids = BTreeSet::new();
+ for process in &processes {
+ if process.ppid != 1 || proc_stat_state_is_dead(process.state) {
+ continue;
+ }
+ let args = proc_cmdline(process.pid);
+ let orphaned_supervisor = cmdline_is_service_supervisor(&args, &current_exe);
+ let orphaned_service = proc_has_managed_service_environment(process.pid, services)
+ && fs::read_link(format!("/proc/{}/cwd", process.pid))
+ .ok()
+ .as_deref()
+ == Some(ROOT.as_path());
+ if orphaned_supervisor || orphaned_service {
+ owned_pids.insert(process.pid);
+ }
+ }
+ loop {
+ let previous_count = owned_pids.len();
+ for process in &processes {
+ if owned_pids.contains(&process.ppid) {
+ owned_pids.insert(process.pid);
+ }
+ }
+ if owned_pids.len() == previous_count {
+ break;
+ }
+ }
+ let group_pids = processes
.iter()
- .map(|spec| format!("target/debug/{}", spec.package))
+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1)
+ .map(|process| process.pgid)
.collect::<BTreeSet<_>>();
- let mut leaders = Vec::new();
+ let leaders = group_pids
+ .into_iter()
+ .map(|pid| RustServiceLeader {
+ pid,
+ members: processes
+ .iter()
+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid)
+ .map(|process| RustServiceProcessIdentity {
+ pid: process.pid,
+ starttime: process.starttime,
+ })
+ .collect(),
+ })
+ .collect::<Vec<_>>();
+ Ok(leaders)
+}
+
+#[cfg(target_os = "linux")]
+fn service_process_snapshot() -> Result<Vec<RustServiceProcess>> {
+ let mut processes = Vec::new();
for entry in fs::read_dir("/proc")? {
let Ok(entry) = entry else {
continue;
@@ -404,18 +506,69 @@ fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServ
else {
continue;
};
- if proc_parent_pid(pid) != Some(1) {
+ let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
continue;
- }
- if !cmdline_has_service_binary(&proc_cmdline(pid), &binaries) {
+ };
+ let Some((state, ppid, pgid, starttime)) = parse_proc_stat_process(&stat) else {
continue;
- }
- if let Some(starttime) = proc_stat_starttime(pid) {
- leaders.push(RustServiceLeader { pid, starttime });
- }
+ };
+ processes.push(RustServiceProcess {
+ pid,
+ ppid,
+ pgid,
+ state,
+ starttime,
+ });
}
- leaders.sort_unstable_by_key(|leader| leader.pid);
- Ok(leaders)
+ Ok(processes)
+}
+
+#[cfg(target_os = "linux")]
+fn cmdline_is_service_supervisor(args: &[String], current_exe: &Path) -> bool {
+ args.first().map(Path::new) == Some(current_exe)
+ && args.get(1).map(String::as_str) == Some("rust-services")
+}
+
+#[cfg(target_os = "linux")]
+fn proc_has_managed_service_environment(pid: i32, services: &[RustServiceSpec]) -> bool {
+ let environment_bytes = fs::read(format!("/proc/{pid}/environ")).unwrap_or_default();
+ let environment = environment_bytes
+ .split(|byte| *byte == 0)
+ .filter_map(|entry| {
+ let separator = entry.iter().position(|byte| *byte == b'=')?;
+ let (key, value) = entry.split_at(separator);
+ Some((key, &value[1..]))
+ })
+ .filter_map(|(key, value)| {
+ Some((
+ std::str::from_utf8(key).ok()?,
+ std::str::from_utf8(value).ok()?,
+ ))
+ })
+ .collect::<std::collections::HashMap<_, _>>();
+ let Some(name) = environment.get("FLUXER_SVC_NAME") else {
+ return false;
+ };
+ let Some(mode) = environment.get("FLUXER_SVC_MODE") else {
+ return false;
+ };
+ let Some(port) = environment
+ .get("FLUXER_SVC_PORT")
+ .and_then(|port| port.parse::<u16>().ok())
+ else {
+ return false;
+ };
+ if environment.get("FLUXER_SVC_LISTEN_HOST") != Some(&"0.0.0.0") {
+ return false;
+ }
+ services.iter().any(|service| {
+ service.name == *name
+ && match *mode {
+ "router" => port == service.port_base,
+ "shard" => port == service.port_base + 1,
+ _ => false,
+ }
+ })
}
#[cfg(target_os = "linux")]
@@ -429,7 +582,7 @@ fn format_pids(pids: &[i32]) -> String {
.join(", ")
}
-#[cfg(any(target_os = "linux", test))]
+#[cfg(test)]
fn cmdline_has_service_binary(args: &[String], binaries: &BTreeSet<String>) -> bool {
args.first()
.map(|arg| binaries.iter().any(|binary| arg.ends_with(binary)))
@@ -446,30 +599,6 @@ fn proc_cmdline(pid: i32) -> Vec<String> {
.collect()
}
-#[cfg(target_os = "linux")]
-fn proc_parent_pid(pid: i32) -> Option<i32> {
- fs::read_to_string(format!("/proc/{pid}/status"))
- .ok()?
- .lines()
- .find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())
-}
-
-#[cfg(target_os = "linux")]
-fn process_exists(pid: i32) -> bool {
- assert!(pid > 0);
- match proc_stat_state_and_pgid(pid) {
- Some((state, _pgid)) => !proc_stat_state_is_dead(state),
- None => false,
- }
-}
-
-#[cfg(target_os = "linux")]
-fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {
- assert!(pid > 0);
- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
- parse_proc_stat_state_and_pgid(&stat)
-}
-
#[cfg(any(target_os = "linux", test))]
fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
let (_, after_comm) = stat.rsplit_once(')')?;
@@ -481,10 +610,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
}
#[cfg(target_os = "linux")]
-fn proc_stat_starttime(pid: i32) -> Option<u64> {
- assert!(pid > 0);
- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
- parse_proc_stat_starttime(&stat)
+fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> {
+ let (_, after_comm) = stat.rsplit_once(')')?;
+ let mut fields = after_comm.split_ascii_whitespace();
+ let state = fields.next()?.chars().next()?;
+ let ppid = fields.next()?.parse().ok()?;
+ let pgid = fields.next()?.parse().ok()?;
+ Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?))
}
#[cfg(any(target_os = "linux", test))]
@@ -500,62 +632,43 @@ fn proc_stat_state_is_dead(state: char) -> bool {
#[cfg(target_os = "linux")]
fn surviving_service_group_pids(leaders: &[RustServiceLeader]) -> Result<Vec<i32>> {
- let leader_pids = leaders
+ Ok(leaders
.iter()
+ .filter(|leader| service_group_has_owned_member(leader))
.map(|leader| leader.pid)
- .collect::<std::collections::HashSet<_>>();
- let mut survivors = Vec::new();
- for entry in fs::read_dir("/proc")? {
- let Ok(entry) = entry else {
- continue;
- };
- let Some(pid) = entry
- .file_name()
- .to_str()
- .and_then(|name| name.parse::<i32>().ok())
- else {
- continue;
- };
- let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
- continue;
+ .collect())
+}
+
+#[cfg(target_os = "linux")]
+fn service_group_has_owned_member(leader: &RustServiceLeader) -> bool {
+ leader.members.iter().any(|member| {
+ let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else {
+ return false;
};
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
- continue;
+ return false;
};
- if proc_stat_state_is_dead(state) {
- continue;
- }
- if leader_pids.contains(&pgid) {
- survivors.push(pid);
- continue;
- }
- let leader = leaders.iter().find(|leader| leader.pid == pid);
- if let Some(leader) = leader
- && parse_proc_stat_starttime(&stat) == Some(leader.starttime)
- {
- survivors.push(pid);
- }
- }
- survivors.sort_unstable();
- Ok(survivors)
+ !proc_stat_state_is_dead(state)
+ && pgid == leader.pid
+ && parse_proc_stat_starttime(&stat) == Some(member.starttime)
+ })
}
#[cfg(target_os = "linux")]
-fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {
+fn signal_process_groups(leaders: &[RustServiceLeader], signal: i32) -> Vec<i32> {
assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);
- let mut failed = Vec::with_capacity(pids.len());
- for pid in pids {
- assert!(*pid > 0);
- let group_result = unsafe { libc::kill(-pid, signal) };
- if group_result == 0 {
+ let mut failed = Vec::with_capacity(leaders.len());
+ for leader in leaders {
+ assert!(leader.pid > 1);
+ if !service_group_has_owned_member(leader) {
continue;
}
- let direct_result = unsafe { libc::kill(*pid, signal) };
- if direct_result == 0 {
+ let group_result = unsafe { libc::kill(-leader.pid, signal) };
+ if group_result == 0 {
continue;
}
- if process_exists(*pid) {
- failed.push(*pid);
+ if service_group_has_owned_member(leader) {
+ failed.push(leader.pid);
}
}
failed
diff --git a/tools/dev/src/tunnel.rs b/tools/dev/src/tunnel.rs
index 6b46840cd..22558f3ed 100644
--- a/tools/dev/src/tunnel.rs
+++ b/tools/dev/src/tunnel.rs
@@ -253,6 +253,16 @@ pub async fn run_cloudflare_tunnel(
return Ok(status.code().unwrap_or(1));
}
if running_inside_devcontainer() {
+ let container = std::fs::read_to_string("/etc/hostname")
+ .context("failed to read the current devcontainer hostname")?;
+ let container = container.trim();
+ if container.is_empty()
+ || !container
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-'))
+ {
+ bail!("Current devcontainer hostname is invalid");
+ }
let docker = docker_command();
let mut command = Command::new(&docker[0]);
command.args(&docker[1..]);
@@ -261,11 +271,7 @@ pub async fn run_cloudflare_tunnel(
"run",
"--rm",
"--network",
- &format!(
- "container:{}",
- std::env::var("HOSTNAME")
- .unwrap_or_else(|_| "fluxer-dev-workspace-1".to_owned())
- ),
+ &format!("container:{container}"),
"cloudflare/cloudflared:latest",
"tunnel",
"run",