A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(workspace): stabilise the development stack (#2024)
Hampus5d2e5932a492
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfileindex f69a4b706..e3c8a9aea 100644--- a/.devcontainer/Dockerfile+++ b/.devcontainer/Dockerfile@@ -7,7 +7,6 @@ ARG USER_UID=1000ARG USER_GID=1000ARG NODE_MAJOR=24ARG ELP_VERSION=2026-02-27-ARG HELM_VERSION=4.2.0ARG PNPM_VERSION=10.29.3ARG WASM_BINDGEN_VERSION=0.2.122@@ -15,8 +14,8 @@ ENV DEBIAN_FRONTEND=noninteractiveRUN apt-get update \&& apt-get install -y --no-install-recommends \+ acl \bash \- brotli \build-essential \ca-certificates \clang \@@ -42,13 +41,13 @@ RUN apt-get update \libfido2-dev \libgbm1 \libgtk-3-0 \+ libimage-exiftool-perl \libnotify4 \libnss3 \libpipewire-0.3-dev \libpulse-dev \libsecret-1-0 \libudev-dev \- libuv1-dev \libcurl4-openssl-dev \libswresample-dev \libswscale-dev \@@ -71,14 +70,12 @@ RUN apt-get update \ninja-build \openssh-client \pkg-config \- protobuf-compiler \python3 \python3-pip \- rsync \+ python3-venv \sudo \rpm \unzip \- webp \xz-utils \xdg-utils \zstd \@@ -90,6 +87,7 @@ RUN apt-get update \bat \btop \docker-cli \+ docker-compose \dnsutils \fd-find \gdb \@@ -122,24 +120,12 @@ RUN apt-get update \&& ln -sf /usr/bin/batcat /usr/local/bin/bat \&& rm -rf /var/lib/apt/lists/*-RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \+RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \&& apt-get install -y --no-install-recommends nodejs \&& rm -rf /var/lib/apt/lists/* \&& corepack enable-RUN ARCH="$(dpkg --print-architecture)" \- && case "$ARCH" in \- amd64) HELM_ARCH="amd64" ;; \- arm64) HELM_ARCH="arm64" ;; \- *) echo "Unsupported architecture for Helm: $ARCH" >&2; exit 1 ;; \- esac \- && curl -fsSL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${HELM_ARCH}.tar.gz" -o /tmp/helm.tgz \- && tar -C /tmp -xzf /tmp/helm.tgz "linux-${HELM_ARCH}/helm" \- && mv "/tmp/linux-${HELM_ARCH}/helm" /usr/local/bin/helm \- && chmod +x /usr/local/bin/helm \- && rm -rf /tmp/helm.tgz "/tmp/linux-${HELM_ARCH}"--RUN python3 -m pip install --break-system-packages --no-cache-dir awscli cqlsh+RUN python3 -m pip install --break-system-packages --no-cache-dir awscliCOPY tools/fonts/requirements.txt /tmp/fluxer-fonts-requirements.txtRUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer-fonts-requirements.txt \@@ -147,18 +133,13 @@ RUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer&& pyftsubset --help >/dev/null \&& python3 -c "import fontTools, brotli"-RUN if ! command -v rebar3 >/dev/null 2>&1; then \- curl -fsSL https://s3.amazonaws.com/rebar3/rebar3 -o /usr/local/bin/rebar3 \- && chmod +x /usr/local/bin/rebar3; \- fi-RUN ARCH="$(dpkg --print-architecture)" \&& case "$ARCH" in \amd64) ELP_ARCH="x86_64" ;; \arm64) ELP_ARCH="aarch64" ;; \*) echo "Unsupported architecture for ELP: $ARCH" >&2; exit 1 ;; \esac \- && curl -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \+ && curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \&& tar -C /usr/local/bin -xzf /tmp/elp.tgz elp \&& chmod +x /usr/local/bin/elp \&& rm /tmp/elp.tgz@@ -179,16 +160,13 @@ ENV DOCKER_HOST="unix:///var/run/docker.sock" \ENV CC_wasm32_unknown_unknown="clang" \AR_wasm32_unknown_unknown="llvm-ar"-RUN curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile default --component clippy,rustfmt \+RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --component clippy,rustfmt \&& rustup target add wasm32-unknown-unknown \&& cargo install cargo-watch --locked \- && cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked+ && cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked \+ && rm -rf "/home/${USERNAME}/.cargo/registry" "/home/${USERNAME}/.cargo/git"RUN corepack prepare "pnpm@${PNPM_VERSION}" --activate \&& pnpm --version-RUN sudo apt-get update \- && sudo apt-get install -y --no-install-recommends python3-venv \- && sudo rm -rf /var/lib/apt/lists/*-WORKDIR /workspaces/fluxerdiff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.jsonindex 16a810fb4..42b866038 100644--- a/.devcontainer/devcontainer.json+++ b/.devcontainer/devcontainer.json@@ -5,20 +5,17 @@"workspaceFolder": "/workspaces/fluxer","shutdownAction": "stopCompose","remoteUser": "vscode",+ "hostRequirements": {+ "cpus": 4,+ "memory": "8gb",+ "storage": "32gb"+ },"remoteEnv": {"DOCKER_HOST": "unix:///var/run/docker.sock"},"runServices": ["workspace", "postgres", "valkey", "nats", "livekit", "meilisearch", "mailpit"],- "forwardPorts": [- 3000, 8088, 8080, 8771, 8082, 3010, 3020, 8100, 8101, 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111,- 8112, 8113, 8114, 8115, 8116, 8117, 8118, 8119, 8120, 8121, 8122, 8123, 8124, 8125, 3900, 8888, 9333, 9340, 23646,- 4222, 7700, 7880, 7900, 9200, 8000- ],+ "forwardPorts": [3000, 8088, 8080, 8771, 8082, 8773, 3010, 3020, 8333],"portsAttributes": {- "8000": {- "label": "Zensical docs",- "onAutoForward": "openBrowserOnce"- },"8088": {"label": "Fluxer dev proxy","onAutoForward": "notify"@@ -29,36 +26,15 @@"3020": {"label": "Fluxer admin"},- "8100": {- "label": "Fluxer Rust service health"- },- "3900": {+ "8333": {"label": "SeaweedFS S3"},- "8888": {- "label": "SeaweedFS filer"- },- "9333": {- "label": "SeaweedFS master"- },- "9340": {- "label": "SeaweedFS volume"- },- "23646": {- "label": "SeaweedFS admin"- },- "7880": {- "label": "LiveKit"- },- "7700": {- "label": "Meilisearch"- },- "9200": {- "label": "Elasticsearch"+ "8773": {+ "label": "Fluxer app proxy"}},- "postCreateCommand": "sudo chown -R vscode:vscode /workspaces/fluxer/target && find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -exec sudo chown -R vscode:vscode {} + && sudo chown -R vscode:vscode /home/vscode/.local/share/pnpm && cargo run -p fluxer-dev -- bootstrap",- "postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && cargo run -p fluxer-dev -- post-start && bash /workspaces/fluxer/fluxer_docs/serve.sh --daemon",+ "postCreateCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- bootstrap",+ "postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- post-start","customizations": {"vscode": {"settings": {diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.ymlindex d6ec6eb44..aec056885 100644--- a/.devcontainer/docker-compose.yml+++ b/.devcontainer/docker-compose.yml@@ -1,5 +1,3 @@-name: fluxer-dev-services:workspace:build:@@ -7,15 +5,29 @@ services:dockerfile: .devcontainer/Dockerfilecommand: sleep infinityinit: true- env_file:- - ../config/env/development.envenvironment:- FLUXER_SEARCH_ENGINE: meilisearch- FLUXER_SEARCH_URL: http://meilisearch:7700- FLUXER_SEARCH_API_KEY: fluxer-dev-meilisearch- FLUXER_POSTGRES_HOST: postgres- FLUXER_SELF_HOSTED: "true"DOCKER_HOST: unix:///var/run/docker.sock+ npm_config_store_dir: /home/vscode/.local/share/pnpm/store+ FLUXER_PUBLIC_PORT: "${FLUXER_DEV_PROXY_PORT:-8088}"+ FLUXER_PUBLIC_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"+ FLUXER_API_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"+ FLUXER_API_CLIENT_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"+ FLUXER_APP_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"+ FLUXER_GATEWAY_ENDPOINT: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/gateway"+ FLUXER_MEDIA_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"+ FLUXER_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"+ FLUXER_ADMIN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin"+ FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"+ FLUXER_S3_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"+ FLUXER_LIVEKIT_URL: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/livekit"+ FLUXER_LIVEKIT_INTERNAL_URL: "http://livekit:7880"+ FLUXER_LIVEKIT_WEBHOOK_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api/webhooks/livekit"+ FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"+ FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"+ FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"+ FLUXER_ADMIN_OAUTH_REDIRECT_URI: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin/oauth2_callback"+ FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: "http://localhost,http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"+ PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"volumes:- ..:/workspaces/fluxer:cached- type: volume@@ -229,6 +241,7 @@ services:volume:nocopy: true- pnpm-store:/home/vscode/.local/share/pnpm/store+ - docs-venv:/workspaces/fluxer/fluxer_docs/.venv- cargo-registry:/home/vscode/.cargo/registry- cargo-git:/home/vscode/.cargo/git- rust-target:/workspaces/fluxer/target@@ -236,45 +249,31 @@ services:source: ${FLUXER_DOCKER_SOCKET:-/var/run/docker.sock}target: /var/run/docker.sockports:- - "${FLUXER_DEV_DOCS_PORT:-8000}:8000"- - "${FLUXER_DEV_RSPACK_PORT:-3000}:3000"- - "${FLUXER_DEV_PROXY_PORT:-8088}:8088"- - "${FLUXER_DEV_APP_PROXY_PORT:-8080}:8080"- - "${FLUXER_DEV_API_PORT:-8771}:8771"- - "${FLUXER_DEV_GATEWAY_PORT:-8082}:8082"- - "${FLUXER_DEV_MARKETING_PORT:-3010}:3010"- - "${FLUXER_DEV_ADMIN_PORT:-3020}:3020"- - "${FLUXER_DEV_RUST_SERVICE_PORTS:-8100-8125}:8100-8125"- - "${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333"- - "${FLUXER_DEV_SEAWEEDFS_FILER_PORT:-8888}:8888"- - "${FLUXER_DEV_SEAWEEDFS_MASTER_PORT:-9333}:9333"- - "${FLUXER_DEV_SEAWEEDFS_VOLUME_PORT:-9340}:9340"- - "${FLUXER_DEV_SEAWEEDFS_ADMIN_PORT:-23646}:23646"+ - "127.0.0.1:${FLUXER_DEV_RSPACK_PORT:-3000}:3000"+ - "127.0.0.1:${FLUXER_DEV_PROXY_PORT:-8088}:8088"+ - "127.0.0.1:${FLUXER_DEV_API_PORT:-8080}:8080"+ - "127.0.0.1:${FLUXER_DEV_GATEWAY_PORT:-8771}:8771"+ - "127.0.0.1:${FLUXER_DEV_MEDIA_PROXY_PORT:-8082}:8082"+ - "127.0.0.1:${FLUXER_DEV_APP_PROXY_PORT:-8773}:8773"+ - "127.0.0.1:${FLUXER_DEV_MARKETING_PORT:-3010}:3010"+ - "127.0.0.1:${FLUXER_DEV_ADMIN_PORT:-3020}:3020"+ - "127.0.0.1:${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333"depends_on:- - postgres- - valkey- - nats- - livekit- - meilisearch- - mailpit+ postgres:+ condition: service_healthy+ valkey:+ condition: service_started+ nats:+ condition: service_started+ livekit:+ condition: service_started+ meilisearch:+ condition: service_healthy+ mailpit:+ condition: service_startedextra_hosts:- "host.docker.internal:host-gateway"- cassandra:- image: cassandra:5.0.8- profiles:- - full- environment:- CASSANDRA_CLUSTER_NAME: fluxer-dev- CASSANDRA_DC: datacenter1- CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch- HEAP_NEWSIZE: 128M- MAX_HEAP_SIZE: 768M- volumes:- - cassandra-data:/var/lib/cassandra- ports:- - "${FLUXER_DEV_CASSANDRA_PORT:-9042}:9042"-postgres:image: postgres:16-alpineenvironment:@@ -284,13 +283,19 @@ services:volumes:- postgres-data:/var/lib/postgresql/dataports:- - "${FLUXER_DEV_POSTGRES_PORT:-5432}:5432"+ - "127.0.0.1:${FLUXER_DEV_POSTGRES_PORT:-5432}:5432"+ healthcheck:+ test: ["CMD-SHELL", "pg_isready -U fluxer -d fluxer"]+ interval: 2s+ timeout: 5s+ retries: 30+ start_period: 5svalkey:image: valkey/valkey:8.1.7-alpinecommand: ["valkey-server", "--save", "", "--appendonly", "no"]ports:- - "${FLUXER_DEV_VALKEY_PORT:-6379}:6379"+ - "127.0.0.1:${FLUXER_DEV_VALKEY_PORT:-6379}:6379"nats:image: nats:2.14.2-alpine@@ -298,33 +303,22 @@ services:volumes:- nats-data:/dataports:- - "${FLUXER_DEV_NATS_PORT:-4222}:4222"- - "${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222"+ - "127.0.0.1:${FLUXER_DEV_NATS_PORT:-4222}:4222"+ - "127.0.0.1:${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222"livekit:image: livekit/livekit-server:v1.12.0command: ["--config", "/etc/livekit.yaml", "--bind", "0.0.0.0"]- volumes:- - ./livekit.yaml:/etc/livekit.yaml:ro- ports:- - "${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880"- - "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:7881"- - "${FLUXER_DEV_LIVEKIT_UDP_PORTS:-7882-7892}:7882-7892/udp"-- elasticsearch:- image: docker.elastic.co/elasticsearch/elasticsearch:9.3.2- profiles:- - fullenvironment:- discovery.type: single-node- xpack.security.enabled: "true"- xpack.security.http.ssl.enabled: "false"- ELASTIC_PASSWORD: fluxer-dev-elasticsearch- ES_JAVA_OPTS: "-Xms512m -Xmx512m"+ LIVEKIT_RTC_TCP_PORT: "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}"+ LIVEKIT_RTC_UDP_PORT_START: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}"+ LIVEKIT_RTC_UDP_PORT_END: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}"volumes:- - elasticsearch-data:/usr/share/elasticsearch/data+ - ./livekit.yaml:/etc/livekit.yaml:roports:- - "${FLUXER_DEV_ELASTICSEARCH_PORT:-9200}:9200"+ - "127.0.0.1:${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880"+ - "127.0.0.1:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}"+ - "127.0.0.1:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}/udp"meilisearch:image: getmeili/meilisearch:v1.12@@ -334,7 +328,13 @@ services:volumes:- meilisearch-data:/meili_dataports:- - "${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700"+ - "127.0.0.1:${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700"+ healthcheck:+ test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:7700/health"]+ interval: 2s+ timeout: 5s+ retries: 30+ start_period: 5smailpit:image: axllent/mailpit:v1.30@@ -349,6 +349,7 @@ services:volumes:pnpm-store:+ docs-venv:root-node-modules:fluxer-api-node-modules:fluxer-app-node-modules:@@ -394,9 +395,7 @@ volumes:cargo-registry:cargo-git:rust-target:- cassandra-data:nats-data:- elasticsearch-data:meilisearch-data:mailpit-data:postgres-data:diff --git a/.devcontainer/fix-docker-socket.sh b/.devcontainer/fix-docker-socket.shindex e0c3fdf59..ea864717f 100755--- a/.devcontainer/fix-docker-socket.sh+++ b/.devcontainer/fix-docker-socket.sh@@ -1,10 +1,10 @@#!/usr/bin/env bash# SPDX-License-Identifier: AGPL-3.0-or-later-set -uo pipefail+set -euo pipefailSOCKET="${DOCKER_SOCKET:-/var/run/docker.sock}"-USER_NAME="${USER:-vscode}"+USER_NAME="$(id -un)"if [ ! -S "$SOCKET" ]; thenecho "fix-docker-socket: no socket at $SOCKET; skipping (Docker-in-devcontainer will not work)"@@ -16,31 +16,11 @@ if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; thenexit 0fi-socket_gid="$(stat -c '%g' "$SOCKET" 2>/dev/null || echo "")"-if [ -z "$socket_gid" ]; then- echo "fix-docker-socket: could not stat $SOCKET; skipping" >&2- exit 0-fi--sudo sh -c '- set -e- gid="$1"- user="$2"- socket="$3"- if ! getent group "$gid" >/dev/null 2>&1; then- groupadd --gid "$gid" docker-host- fi- group_name="$(getent group "$gid" | cut -d: -f1)"- usermod --append --groups "$group_name" "$user"- chgrp "$gid" "$socket"- chmod g+rw "$socket"-' sh "$socket_gid" "$USER_NAME" "$SOCKET" || {- echo "fix-docker-socket: could not adjust $SOCKET; run docker with sudo" >&2- exit 0-}+sudo setfacl --modify "user:${USER_NAME}:rw" "$SOCKET"if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then- echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME (gid $socket_gid)"+ echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME"else- echo "fix-docker-socket: $SOCKET still unreachable as $USER_NAME; run docker with sudo" >&2+ echo "fix-docker-socket: $SOCKET is still unreachable as $USER_NAME" >&2+ exit 1fidiff --git a/.devcontainer/fix-workspace-permissions.sh b/.devcontainer/fix-workspace-permissions.shnew file mode 100644index 000000000..0ef0c5d4e--- /dev/null+++ b/.devcontainer/fix-workspace-permissions.sh@@ -0,0 +1,40 @@+#!/usr/bin/env bash+# SPDX-License-Identifier: AGPL-3.0-or-later++set -euo pipefail++owner="$(id -u):$(id -g)"++repair_tree() {+ local path="$1"+ local unwritable+ if [ ! -d "$path" ]; then+ echo "fix-workspace-permissions: expected mount is missing: $path" >&2+ exit 1+ fi+ unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)"+ if [ ! -w "$path" ] || [ -n "$unwritable" ]; then+ sudo find "$path" -xdev \( -type d -o -type f \) -exec chown "$owner" {} ++ fi+ unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)"+ if [ ! -w "$path" ] || [ -n "$unwritable" ]; then+ echo "fix-workspace-permissions: $path is not writable as $(id -un)" >&2+ exit 1+ fi+}++for path in \+ /workspaces/fluxer/target \+ /home/vscode/.cargo/registry \+ /home/vscode/.cargo/git \+ /home/vscode/.local \+ /home/vscode/.local/share/pnpm/store \+ /workspaces/fluxer/fluxer_docs/.venv; do+ repair_tree "$path"+done++while IFS= read -r -d '' path; do+ if mountpoint -q "$path"; then+ repair_tree "$path"+ fi+done < <(find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -print0)diff --git a/.devcontainer/livekit.yaml b/.devcontainer/livekit.yamlindex 552df10d2..f49d40c42 100644--- a/.devcontainer/livekit.yaml+++ b/.devcontainer/livekit.yaml@@ -1,11 +1,10 @@port: 7880keys:- devkey: secret+ devkey: fluxer-livekit-development-secretrtc:tcp_port: 7881- udp_port: 7882-7892node_ip: 127.0.0.1use_mdns: truestun_servers:diff --git a/config/env/development.env b/config/env/development.envindex 5cf6d0547..7a4867130 100644--- a/config/env/development.env+++ b/config/env/development.env@@ -30,12 +30,6 @@ FLUXER_POSTGRES_PASSWORD=fluxerFLUXER_POSTGRES_SSL=falseFLUXER_POSTGRES_MAX_CONNECTIONS=20FLUXER_POSTGRES_KV_TABLE=fluxer_kv-FLUXER_CASSANDRA_HOSTS=cassandra-FLUXER_CASSANDRA_PORT=9042-FLUXER_CASSANDRA_KEYSPACE=fluxer-FLUXER_CASSANDRA_LOCAL_DC=datacenter1-FLUXER_CASSANDRA_USERNAME=fluxer-FLUXER_CASSANDRA_PASSWORD=fluxerFLUXER_KV_URL=redis://valkey:6379/0FLUXER_NATS_URL=nats://nats:4222FLUXER_NATS_JETSTREAM_URL=nats://nats:4222@@ -66,8 +60,9 @@ FLUXER_S3_BUCKET_STATIC=fluxer-staticFLUXER_LIVEKIT_ENABLED=trueFLUXER_LIVEKIT_URL=ws://localhost:8088/livekit+FLUXER_LIVEKIT_INTERNAL_URL=http://localhost:7880FLUXER_LIVEKIT_API_KEY=devkey-FLUXER_LIVEKIT_API_SECRET=secret+FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secretFLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekitFLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686}diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.ymlindex d3484fe61..57fc260ad 100644--- a/deploy/self-hosting/docker-compose.yml+++ b/deploy/self-hosting/docker-compose.yml@@ -47,6 +47,7 @@ x-fluxer-env: &fluxer-envFLUXER_LIVEKIT_ENABLED: "true"FLUXER_LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}+ FLUXER_LIVEKIT_INTERNAL_URL: http://livekit:7880FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekitFLUXER_LIVEKIT_DEFAULT_REGION: '{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}'diff --git a/fluxer_api/src/api/App.ts b/fluxer_api/src/api/App.tsindex a35882ba5..9d982b505 100644--- a/fluxer_api/src/api/App.ts+++ b/fluxer_api/src/api/App.ts@@ -39,7 +39,7 @@ function AbuseAwareAppErrorHandler(err: Error, ctx: Context<HonoEnv>): Responseexport async function createAPIApp(options: CreateAPIAppOptions): Promise<APIAppResult> {const {config, logger} = options;- const shutdownApiLifecycle = createShutdown(logger);+ const shutdownApiLifecycle = createShutdown(config, logger);setIsDevelopment(config.nodeEnv === 'development');const routes = new Hono<HonoEnv>({strict: true});configureMiddleware(routes, {diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.tsindex e32b81674..422a4158e 100644--- a/fluxer_api/src/api/Config.ts+++ b/fluxer_api/src/api/Config.ts@@ -306,6 +306,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {apiSecret: master.integrations.voice.api_secret,webhookUrl: master.integrations.voice.webhook_url,url: master.integrations.voice.url,+ internalUrl: master.integrations.voice.internal_url,defaultRegion: master.integrations.voice.default_region,},stripe: {diff --git a/fluxer_api/src/api/app/APILifecycle.ts b/fluxer_api/src/api/app/APILifecycle.tsindex 2cfd66fee..9097416d0 100644--- a/fluxer_api/src/api/app/APILifecycle.ts+++ b/fluxer_api/src/api/app/APILifecycle.ts@@ -36,6 +36,11 @@ import {JetStreamWorkerQueue} from '../worker/JetStreamWorkerQueue';import {WorkerService} from '../worker/WorkerService';let jsConnectionManager: JetStreamConnectionManager | null = null;++function unsupportedDatabaseBackend(backend: never): never {+ throw new Error(`Unsupported database backend during shutdown: ${String(backend)}`);+}+export function createInitializer(config: APIConfig, logger: ILogger): () => Promise<void> {return async (): Promise<void> => {try {@@ -131,7 +136,7 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Protry {const kvDeletionQueue = getKVAccountDeletionQueue();if (await kvDeletionQueue.needsRebuild()) {- logger.warn('KV deletion queue needs rebuild, rebuilding...');+ logger.info('KV deletion queue needs rebuild, rebuilding...');await kvDeletionQueue.rebuildState();} else {logger.info('KV deletion queue state is healthy');@@ -204,13 +209,13 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Prologger.info('API service initialization complete');} catch (error) {logger.error({error}, 'API service initialization failed');- await createShutdown(logger)();+ await createShutdown(config, logger)();throw error;}};}-export function createShutdown(logger: ILogger): () => Promise<void> {+export function createShutdown(config: APIConfig, logger: ILogger): () => Promise<void> {return async (): Promise<void> => {logger.info('Shutting down API service...');if (jsConnectionManager) {@@ -258,18 +263,26 @@ export function createShutdown(logger: ILogger): () => Promise<void> {} catch (error) {logger.error({error}, 'Error shutting down report service');}- try {- setDatabaseQueryExecutor(null);- await shutdownPostgres();- logger.info('Postgres client shut down');- } catch (error) {- logger.error({error}, 'Error shutting down Postgres client');- }- try {- await shutdownCassandra();- logger.info('Cassandra client shut down');- } catch (error) {- logger.error({error}, 'Error shutting down Cassandra client');+ setDatabaseQueryExecutor(null);+ switch (config.database.backend) {+ case 'postgres':+ try {+ await shutdownPostgres();+ logger.info('Postgres client shut down');+ } catch (error) {+ logger.error({error}, 'Error shutting down Postgres client');+ }+ break;+ case 'cassandra':+ try {+ await shutdownCassandra();+ logger.info('Cassandra client shut down');+ } catch (error) {+ logger.error({error}, 'Error shutting down Cassandra client');+ }+ break;+ default:+ unsupportedDatabaseBackend(config.database.backend);}logger.info('API service shutdown complete');};diff --git a/fluxer_api/src/api/app/ControllerRegistry.ts b/fluxer_api/src/api/app/ControllerRegistry.tsindex 4db31caa0..28c851d9b 100644--- a/fluxer_api/src/api/app/ControllerRegistry.ts+++ b/fluxer_api/src/api/app/ControllerRegistry.ts@@ -66,7 +66,9 @@ export function registerControllers(routes: HonoApp, config: APIConfig): void {TestHarnessController(routes);}UserController(routes);- registerInboundSmsWebhook(routes);+ if (config.sms.enabled) {+ registerInboundSmsWebhook(routes);+ }WebhookController(routes);OAuth2Controller(routes);OAuth2ApplicationsController(routes);diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.tsindex 3b685ce92..58ea5fd50 100644--- a/fluxer_api/src/api/config/APIConfig.ts+++ b/fluxer_api/src/api/config/APIConfig.ts@@ -204,6 +204,7 @@ export interface APIConfig {apiSecret?: string;webhookUrl?: string;url?: string;+ internalUrl?: string;defaultRegion?: {id: string;name: string;diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.tsindex db05aea5e..8ae05ca42 100644--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts@@ -19,6 +19,8 @@ interface PageState {const VALUE_SEPARATOR = '\u001f';const ENCODED_TYPE_KEY = '__fluxer_type';+const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;+const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';function normalizeCql(cql: string): string {return cql.replace(/\s+/g, ' ').trim();@@ -354,8 +356,13 @@ function parseEqWhere(whereSql: string, cql: string): ReadonlyArray<EqWhereExpr>}export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<void> {- const table = quoteIdentifier(client.kvTable());- await client.query(`+ const kvTable = client.kvTable();+ const table = quoteIdentifier(kvTable);+ await client.transaction(async (db) => {+ await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_LOCK_TIMEOUT]);+ await db.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, kvTable]);+ await db.query("SELECT set_config('statement_timeout', '0', true)");+ await db.query(`CREATE TABLE IF NOT EXISTS ${table} (table_name text NOT NULL,partition_key text NOT NULL,@@ -365,28 +372,29 @@ CREATE TABLE IF NOT EXISTS ${table} (updated_at timestamptz NOT NULL DEFAULT now(),PRIMARY KEY (table_name, row_key))`);- await client.query(- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,- );- await client.query(- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,- );- await client.query(- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,- );- await client.query(- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,- );- await client.query(- `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,- );- await client.query(`+ await db.query(+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,+ );+ await db.query(+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,+ );+ await db.query(+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,+ );+ await db.query(+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,+ );+ await db.query(+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,+ );+ await db.query(`UPDATE ${table}SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)WHERE table_name = 'messages'AND partition_key = row_keyAND split_part(row_key, chr(31), 3) <> ''`);- await client.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_idx`)}`);+ await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);+ });}export async function pruneExpiredPostgresKvRows(client: IPostgresClient, batchSize = 5000): Promise<number> {diff --git a/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts b/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.tsindex ec0174ad9..661175056 100644--- a/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts+++ b/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts@@ -60,7 +60,7 @@ export class KVAccountDeletionQueueService {}async rebuildState(): Promise<void> {- Logger.info('Starting deletion queue rebuild from Cassandra');+ Logger.info('Starting deletion queue rebuild from primary database');try {await this.kvClient.del(QUEUE_KEY);await this.kvClient.del(STATE_VERSION_KEY);diff --git a/fluxer_api/src/api/infrastructure/LiveKitService.ts b/fluxer_api/src/api/infrastructure/LiveKitService.tsindex 8723c1443..597915364 100644--- a/fluxer_api/src/api/infrastructure/LiveKitService.ts+++ b/fluxer_api/src/api/infrastructure/LiveKitService.ts@@ -109,6 +109,18 @@ function createRoomServiceClient(endpoint: string, apiKey: string, apiSecret: streturn client;}+function resolveRoomServiceEndpoint(server: VoiceServerRecord): string {+ const defaultRegion = Config.voice.defaultRegion;+ const internalUrl = Config.voice.internalUrl;+ if (!defaultRegion || !internalUrl) {+ return server.endpoint;+ }+ if (server.regionId !== defaultRegion.id || server.serverId !== `${defaultRegion.id}-server-1`) {+ return server.endpoint;+ }+ return internalUrl;+}+export class LiveKitService extends ILiveKitService {private serverClients: Map<string, Map<string, ServerClientConfig>> = new Map();private topology: VoiceTopology;@@ -453,12 +465,13 @@ export class LiveKitService extends ILiveKitService {const servers = this.topology.getServersForRegion(region.id);const serverMap: Map<string, ServerClientConfig> = new Map();for (const server of servers) {+ const roomServiceEndpoint = resolveRoomServiceEndpoint(server);serverMap.set(server.serverId, {endpoint: server.endpoint,apiKey: server.apiKey,apiSecret: server.apiSecret,isActive: server.isActive,- roomServiceClient: createRoomServiceClient(server.endpoint, server.apiKey, server.apiSecret),+ roomServiceClient: createRoomServiceClient(roomServiceEndpoint, server.apiKey, server.apiSecret),});}newMap.set(region.id, serverMap);diff --git a/fluxer_api/src/api/middleware/ServiceMiddleware.ts b/fluxer_api/src/api/middleware/ServiceMiddleware.tsindex 5fb8945aa..ca44b7db2 100644--- a/fluxer_api/src/api/middleware/ServiceMiddleware.ts+++ b/fluxer_api/src/api/middleware/ServiceMiddleware.ts@@ -284,7 +284,7 @@ export function setInjectedAccountPolicyEvaluator(evaluator: IAccountPolicyEvalufunction getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {if (_registrationRiskEvaluator) return _registrationRiskEvaluator;if (!Config.risk.enabled) {- Logger.warn(+ Logger.info({},'[ServiceMiddleware] integrations.risk_integration.enabled is false — account risk scoring is disabled',);diff --git a/fluxer_app_proxy/src/routes/assets_proxy.rs b/fluxer_app_proxy/src/routes/assets_proxy.rsindex 7d516d388..c2ed95abc 100644--- a/fluxer_app_proxy/src/routes/assets_proxy.rs+++ b/fluxer_app_proxy/src/routes/assets_proxy.rs@@ -148,7 +148,7 @@ pub async fn proxy_assets(response}-async fn serve_local_asset(+pub(super) async fn serve_local_asset(static_dir: &str,relative_path: &str,request_headers: &HeaderMap,diff --git a/fluxer_app_proxy/src/routes/spa_index.rs b/fluxer_app_proxy/src/routes/spa_index.rsindex 487765422..96cfce35d 100644--- a/fluxer_app_proxy/src/routes/spa_index.rs+++ b/fluxer_app_proxy/src/routes/spa_index.rs@@ -19,6 +19,7 @@ use axum::{use std::path::Path;use std::time::{Duration, SystemTime, UNIX_EPOCH};+use super::assets_proxy::serve_local_asset;use super::spa_static::{CORS_ALLOW_ANY_VALUE, guess_mime, is_font_mime};const ACCEPT_CH_VALUE: &str = "DPR, Sec-CH-DPR, Sec-CH-Width, Save-Data, ECT, Downlink";@@ -35,6 +36,14 @@ pub async fn spa_catch_all(if let Some(cache_control) = static_root_file_cache_control(request_path) {return serve_static_file(&state.config.static_dir, request_path, cache_control).await;}+ if is_static_asset_path(request_path) {+ return serve_local_asset(+ &state.config.static_dir,+ request_path.trim_start_matches('/'),+ &headers,+ )+ .await;+ }serve_spa_index(&state, &headers, request_path).await}@@ -42,6 +51,16 @@ pub async fn spa_catch_all(const CRAWL_CONTROL_CACHE_CONTROL: &str = "public, max-age=300, must-revalidate";const STATIC_ROOT_FILES: &[(&str, &str)] = &[("/robots.txt", CRAWL_CONTROL_CACHE_CONTROL)];+const STATIC_ASSET_PREFIXES: &[&str] = &[+ "/avatars/",+ "/badges/",+ "/desktop/",+ "/embeds/",+ "/emoji/",+ "/libs/",+ "/marketing/",+ "/web/",+];fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> {STATIC_ROOT_FILES@@ -50,6 +69,12 @@ fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> {.map(|(_, cache_control)| *cache_control)}+fn is_static_asset_path(request_path: &str) -> bool {+ STATIC_ASSET_PREFIXES+ .iter()+ .any(|prefix| request_path.starts_with(prefix))+}+async fn serve_static_file(static_dir: &str,request_path: &str,diff --git a/fluxer_docs/serve.sh b/fluxer_docs/serve.shindex 8cb5c270a..d744c9067 100755--- a/fluxer_docs/serve.sh+++ b/fluxer_docs/serve.sh@@ -3,6 +3,7 @@ set -euo pipefailHERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"VENV="$HERE/.venv"+REQUIREMENTS_STAMP="$VENV/.requirements.sha256"ADDR="${ZENSICAL_DEV_ADDR:-0.0.0.0:8000}"LOG="${ZENSICAL_LOG:-/tmp/zensical-serve.log}"PORT="${ADDR##*:}"@@ -14,29 +15,34 @@ ensure_env() {python3 -m venv "$VENV""$VENV/bin/python" -m pip install --quiet --upgrade pipfi- "$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt"+ python_fingerprint="$(python3 -c 'import platform, sys; print(f"{sys.implementation.cache_tag}:{platform.machine()}")')"+ requirements_hash="$(printf '%s\0%s\n' "$python_fingerprint" "$(sha256sum "$HERE/requirements.txt" | cut -d' ' -f1)" | sha256sum | cut -d' ' -f1)"+ if [ ! -f "$REQUIREMENTS_STAMP" ] || [ "$(cat "$REQUIREMENTS_STAMP")" != "$requirements_hash" ]; then+ "$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt"+ printf '%s\n' "$requirements_hash" >"$REQUIREMENTS_STAMP"+ fi}case "${1:-serve}" in- --bootstrap)- ensure_env- ;;- --daemon)- ensure_env- if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then- echo "zensical already serving on ${ADDR}"- exit 0- fi- setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 &- disown 2>/dev/null || true- echo "zensical serving on ${ADDR} (logs: ${LOG})"- ;;- serve)- ensure_env- exec "$VENV/bin/zensical" serve -a "$ADDR"- ;;- *)- ensure_env- exec "$VENV/bin/zensical" "$@"- ;;+--bootstrap)+ ensure_env+ ;;+--daemon)+ ensure_env+ if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then+ echo "zensical already serving on ${ADDR}"+ exit 0+ fi+ setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 &+ disown 2>/dev/null || true+ echo "zensical serving on ${ADDR} (logs: ${LOG})"+ ;;+serve)+ ensure_env+ exec "$VENV/bin/zensical" serve -a "$ADDR"+ ;;+*)+ ensure_env+ exec "$VENV/bin/zensical" "$@"+ ;;esacdiff --git a/fluxer_gateway/Dockerfile b/fluxer_gateway/Dockerfileindex 0e715a505..32ff1da63 100644--- a/fluxer_gateway/Dockerfile+++ b/fluxer_gateway/Dockerfile@@ -12,22 +12,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \git \curl \make \- cmake \gcc \g++ \libc6-dev \- libssl-dev \- libuv1-dev \- zlib1g-dev \- pkg-config \gettext-base \ca-certificates \&& rm -rf /var/lib/apt/lists/*-RUN curl -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \+RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \chmod +x /usr/local/bin/rebar3-RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \+RUN curl --retry 5 --retry-delay 2 --retry-all-errors --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}"COPY . .@@ -47,8 +42,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ca-certificates \curl \libstdc++6 \- libuv1t64 \- zlib1g \&& rm -rf /var/lib/apt/lists/*COPY --from=build /usr/src/app/fluxer_gateway/_build/prod/rel/fluxer_gateway .diff --git a/fluxer_gateway/rebar.config b/fluxer_gateway/rebar.configindex 99a565e7b..de42094df 100644--- a/fluxer_gateway/rebar.config+++ b/fluxer_gateway/rebar.config@@ -6,7 +6,6 @@{jose, "1.11.10"},{ezstd, "1.1.0"},{enats, "1.2.0"},- {erlcass, "4.1.3"},{eqwalizer_support,{git_subdir, "https://github.com/whatsapp/eqwalizer.git",{ref, "c57aa9e3a05553d1ba66ea647a2177eaa469211b"}, "eqwalizer_support"}}@@ -32,7 +31,6 @@{relx, [{release, {fluxer_gateway, "0.0.0"}, [fluxer_gateway,- {erlcass, load},enats,sasl]},@@ -90,7 +88,7 @@]},{plt_apps, all_deps},{plt_extra_apps, [- crypto, enats, erlcass, inets, jose, public_key, ranch, ssl+ crypto, enats, inets, jose, public_key, ranch, ssl]},{warnings_file, "dialyzer.ignore-warnings"}]}.diff --git a/fluxer_gateway/rebar.lock b/fluxer_gateway/rebar.lockindex 2e2d2b7bf..e193fe0ff 100644--- a/fluxer_gateway/rebar.lock+++ b/fluxer_gateway/rebar.lock@@ -9,7 +9,6 @@{ref,"c57aa9e3a05553d1ba66ea647a2177eaa469211b"},"eqwalizer_support"},0},- {<<"erlcass">>,{pkg,<<"erlcass">>,<<"4.1.3">>},0},{<<"ezstd">>,{pkg,<<"ezstd">>,<<"1.1.0">>},0},{<<"jose">>,{pkg,<<"jose">>,<<"1.11.10">>},0},{<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.4.0">>},1},@@ -21,7 +20,6 @@{<<"cowlib">>, <<"54592074EBBBB92EE4746C8A8846E5605052F29309D3A873468D76CDF932076F">>},{<<"enats">>, <<"D7459C804013CAFA4AF880B18D446C48890D28D372D62AD66C76187E5779248D">>},{<<"enats_msg">>, <<"50631124F37D88BE76A91A5B96A6565C5981EBF917CD819F0175CA658A966F43">>},- {<<"erlcass">>, <<"17AC8F39B0A4703B12A8994663920993E191757831E125CF2C7EFB267DB66E7D">>},{<<"ezstd">>, <<"D3B483D6ACFADFB65DBA4015371E6D54526DBF3D9EF0941B5ADD8BF5890731F4">>},{<<"jose">>, <<"A903F5227417BD2A08C8A00A0CBCC458118BE84480955E8D251297A425723F83">>},{<<"opentelemetry_api">>, <<"63CA1742F92F00059298F478048DFB826F4B20D49534493D6919A0DB39B6DB04">>},@@ -32,7 +30,6 @@{<<"cowlib">>, <<"7F478D80D66B747344F0EA7708C187645CFCC08B11AA424632F78E25BF05DB51">>},{<<"enats">>, <<"20DEB3CB1D3E960194DF8B136C40D2DB085B485BBA5E493B340AB5F9FD2BED22">>},{<<"enats_msg">>, <<"C4F2139E5144FABC99AFE01B8B016AD9DA278CDDC60857AC5D5AFB0AD1283534">>},- {<<"erlcass">>, <<"37237C12ABE5745E0F900180D0ADB129634619BD6B2F57995BA020E63DDC6BEC">>},{<<"ezstd">>, <<"28CFA0ED6CC3922095AD5BA0F23392A1664273358B17184BAA909868361184E7">>},{<<"jose">>, <<"0D6CD36FF8BA174DB29148FC112B5842186B68A90CE9FC2B3EC3AFE76593E614">>},{<<"opentelemetry_api">>, <<"3DFBBFAA2C2ED3121C5C483162836C4F9027DEF469C41578AF5EF32589FCFC58">>},diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl b/fluxer_gateway/src/gateway/fluxer_gateway_config.erlindex 81ec09b4f..64c7b6682 100644--- a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl+++ b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl@@ -59,7 +59,7 @@ env_services_config() ->-spec env_gateway_config() -> map().env_gateway_config() ->- maps:merge(env_gateway_base_config(), env_gateway_hotpatch_config()).+ env_gateway_base_config().-spec env_gateway_base_config() -> map().env_gateway_base_config() ->@@ -107,36 +107,6 @@ env_gateway_base_config() ->)}.--spec env_gateway_hotpatch_config() -> map().-env_gateway_hotpatch_config() ->- #{- <<"hotpatch_enabled">> => env_bool("FLUXER_GATEWAY_HOTPATCH_ENABLED", false),- <<"hotpatch_cassandra_hosts">> => env_optional_binary(- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_HOSTS"- ),- <<"hotpatch_cassandra_port">> => env_int(- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PORT", 9042- ),- <<"hotpatch_cassandra_keyspace">> => env_binary(- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_KEYSPACE", <<"fluxer">>- ),- <<"hotpatch_cassandra_username">> => env_optional_binary(- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_USERNAME"- ),- <<"hotpatch_cassandra_password">> => env_optional_binary(- "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PASSWORD"- ),- <<"hotpatch_public_keys">> => env_optional_binary(- "FLUXER_GATEWAY_HOTPATCH_PUBLIC_KEYS"- ),- <<"hotpatch_poll_interval_ms">> => env_int(- "FLUXER_GATEWAY_HOTPATCH_POLL_INTERVAL_MS", 5000- ),- <<"hotpatch_startup_sync_timeout_ms">> => env_int(- "FLUXER_GATEWAY_HOTPATCH_STARTUP_SYNC_TIMEOUT_MS", 30000- )- }.--spec env_nats_config() -> map().env_nats_config() ->#{@@ -295,20 +265,6 @@ build_http_config(Service) ->),gateway_http_recovery_timeout_ms =>get_int(Service, <<"gateway_http_recovery_timeout_ms">>, 15000),- hotpatch_enabled => get_bool(Service, <<"hotpatch_enabled">>, false),- hotpatch_cassandra_hosts =>- optional_string(get_optional_binary(Service, <<"hotpatch_cassandra_hosts">>)),- hotpatch_cassandra_port => get_int(Service, <<"hotpatch_cassandra_port">>, 9042),- hotpatch_cassandra_keyspace =>- get_binary(Service, <<"hotpatch_cassandra_keyspace">>, <<"fluxer">>),- hotpatch_cassandra_username =>- get_optional_binary(Service, <<"hotpatch_cassandra_username">>),- hotpatch_cassandra_password =>- get_optional_binary(Service, <<"hotpatch_cassandra_password">>),- hotpatch_public_keys => get_optional_binary(Service, <<"hotpatch_public_keys">>),- hotpatch_poll_interval_ms => get_int(Service, <<"hotpatch_poll_interval_ms">>, 5000),- hotpatch_startup_sync_timeout_ms =>- get_int(Service, <<"hotpatch_startup_sync_timeout_ms">>, 30000),gateway_http_cleanup_interval_ms =>get_int(Service, <<"gateway_http_cleanup_interval_ms">>, 30000),gateway_http_cleanup_max_age_ms =>diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erlindex 1bea7e15d..e2b953955 100644--- a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl+++ b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl@@ -38,8 +38,7 @@ common_children() ->child_spec(gateway_nats_pool, gateway_nats_pool),child_spec(gateway_event_pause, gateway_event_pause),child_spec(gateway_concurrency, gateway_concurrency),- child_spec(gateway_rollout_config, gateway_rollout_config),- child_spec(gateway_hotpatch_reconciler, gateway_hotpatch_reconciler)+ child_spec(gateway_rollout_config, gateway_rollout_config)] ++ cluster_children() ++[child_spec(gateway_dispatch_relay, gateway_dispatch_relay),diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erldeleted file mode 100644index 401595834..000000000--- a/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl+++ /dev/null@@ -1,292 +0,0 @@-%% SPDX-License-Identifier: AGPL-3.0-or-later---module(gateway_hotpatch_bundle).--typing([eqwalizer]).---export([- compress_term/1,- decompress_term/1,- bundle_hash/1,- signing_payload/1,- sign/2,- verify_signature/4,- parse_public_keys/1,- decode_signed_event/2-]).---ifdef(TEST).--include_lib("eunit/include/eunit.hrl").--endif.---define(DOMAIN, <<"fluxer-gateway-hotpatch-v1">>).--define(ZSTD_LEVEL, 3).---spec compress_term(term()) -> {ok, binary()} | {error, term()}.-compress_term(Term) ->- compress_binary(term_to_binary(Term, [deterministic])).---spec decompress_term(binary()) -> {ok, term()} | {error, term()}.-decompress_term(Compressed) when is_binary(Compressed) ->- case decompress_binary(Compressed) of- {ok, Binary} -> decode_term(Binary);- {error, Reason} -> {error, Reason}- end.---spec decode_term(binary()) -> {ok, term()} | {error, term()}.-decode_term(Binary) ->- try- {ok, binary_to_term(Binary, [safe])}- catch- Class:Reason -> {error, {invalid_term, Class, Reason}}- end.---spec bundle_hash(binary()) -> binary().-bundle_hash(CompressedBundle) when is_binary(CompressedBundle) ->- crypto:hash(sha256, CompressedBundle).---spec signing_payload(binary()) -> binary().-signing_payload(CompressedBundle) when is_binary(CompressedBundle) ->- <<?DOMAIN/binary, 0, CompressedBundle/binary>>.---spec sign(binary(), binary()) -> {ok, binary()} | {error, term()}.-sign(CompressedBundle, PrivateKey) when is_binary(CompressedBundle), is_binary(PrivateKey) ->- try- {ok, crypto:sign(eddsa, none, signing_payload(CompressedBundle), [PrivateKey, ed25519])}- catch- Class:Reason -> {error, {sign_failed, Class, Reason}}- end.---spec verify_signature(binary(), binary(), binary(), #{binary() => binary()}) ->- ok | {error, term()}.-verify_signature(CompressedBundle, SignerKeyId, Signature, PublicKeys) when- is_binary(CompressedBundle),- is_binary(SignerKeyId),- is_binary(Signature),- is_map(PublicKeys)-->- case maps:get(SignerKeyId, PublicKeys, undefined) of- undefined ->- {error, {unknown_signer, SignerKeyId}};- PublicKey ->- verify_with_key(CompressedBundle, Signature, PublicKey)- end.---spec parse_public_keys(term()) ->- {ok, #{binary() => binary()}} | {error, term()}.-parse_public_keys(undefined) ->- {ok, #{}};-parse_public_keys(Value) when is_list(Value) ->- parse_public_keys(type_conv:ensure_binary(Value));-parse_public_keys(Value) when is_binary(Value) ->- Tokens = [- string:trim(Token)- || Token <- binary:split(Value, [<<",">>, <<"\n">>, <<";">>], [global]),- string:trim(Token) =/= <<>>- ],- parse_public_key_tokens(Tokens, #{});-parse_public_keys(Value) ->- {error, {invalid_public_keys_config, Value}}.---spec decode_signed_event(map(), #{binary() => binary()}) -> {ok, map()} | {error, term()}.-decode_signed_event(Event, PublicKeys) when is_map(Event), is_map(PublicKeys) ->- Bundle = event_value(bundle, Event),- SignerKeyId = event_value(signer_key_id, Event),- Signature = event_value(signature, Event),- BundleHash = event_value(bundle_sha256, Event),- decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys).---spec decode_event_payload(term(), term(), term(), term(), #{binary() => binary()}) ->- {ok, map()} | {error, term()}.-decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys) when- is_binary(Bundle), is_binary(SignerKeyId), is_binary(Signature), is_binary(BundleHash)-->- case validate_event_hash(Bundle, BundleHash) of- ok -> decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys);- {error, Reason} -> {error, Reason}- end;-decode_event_payload(_Bundle, _SignerKeyId, _Signature, _BundleHash, _PublicKeys) ->- {error, invalid_event_payload}.---spec decode_verified_event(binary(), binary(), binary(), #{binary() => binary()}) ->- {ok, map()} | {error, term()}.-decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys) ->- case verify_signature(Bundle, SignerKeyId, Signature, PublicKeys) of- ok -> decompress_bundle_map(Bundle);- {error, Reason} -> {error, Reason}- end.---spec decompress_bundle_map(binary()) -> {ok, map()} | {error, term()}.-decompress_bundle_map(Bundle) ->- case decompress_term(Bundle) of- {ok, Map} when is_map(Map) -> {ok, Map};- {ok, Other} -> {error, {invalid_bundle_term, Other}};- {error, Reason} -> {error, Reason}- end.---spec compress_binary(binary()) -> {ok, binary()} | {error, term()}.-compress_binary(Binary) ->- try erlang:apply(ezstd, compress, [Binary, ?ZSTD_LEVEL]) of- Compressed when is_binary(Compressed) -> {ok, Compressed};- {error, Reason} -> {error, {compress_failed, Reason}};- Other -> {error, {compress_failed, Other}}- catch- Class:Reason -> {error, {compress_failed, Class, Reason}}- end.---spec decompress_binary(binary()) -> {ok, binary()} | {error, term()}.-decompress_binary(Binary) ->- try erlang:apply(ezstd, decompress, [Binary]) of- Decompressed when is_binary(Decompressed) -> {ok, Decompressed};- Decompressed when is_list(Decompressed) -> {ok, iolist_to_binary(Decompressed)};- {error, Reason} -> {error, {decompress_failed, Reason}};- Other -> {error, {decompress_failed, Other}}- catch- Class:Reason -> {error, {decompress_failed, Class, Reason}}- end.---spec verify_with_key(binary(), binary(), binary()) -> ok | {error, term()}.-verify_with_key(CompressedBundle, Signature, PublicKey) when byte_size(PublicKey) =:= 32 ->- verify_with_valid_key(CompressedBundle, Signature, PublicKey);-verify_with_key(_CompressedBundle, _Signature, PublicKey) ->- {error, {invalid_public_key_size, byte_size(PublicKey)}}.---spec verify_with_valid_key(binary(), binary(), binary()) -> ok | {error, term()}.-verify_with_valid_key(CompressedBundle, Signature, PublicKey) ->- try- crypto:verify(eddsa, none, signing_payload(CompressedBundle), Signature, [- PublicKey, ed25519- ])- of- true -> ok;- false -> {error, invalid_signature}- catch- Class:Reason -> {error, {verify_failed, Class, Reason}}- end.---spec parse_public_key_tokens([binary()], #{binary() => binary()}) ->- {ok, #{binary() => binary()}} | {error, term()}.-parse_public_key_tokens([], Acc) ->- {ok, Acc};-parse_public_key_tokens([Token | Rest], Acc) ->- case parse_public_key_token(Token) of- {ok, KeyId, PublicKey} -> parse_public_key_tokens(Rest, Acc#{KeyId => PublicKey});- {error, Reason} -> {error, Reason}- end.---spec parse_public_key_token(binary()) -> {ok, binary(), binary()} | {error, term()}.-parse_public_key_token(Token) ->- case split_key_token(Token) of- {ok, KeyId, Encoded} -> parse_public_key_material(KeyId, Encoded);- error -> {error, {invalid_public_key_token, Token}}- end.---spec parse_public_key_material(binary(), binary()) ->- {ok, binary(), binary()} | {error, term()}.-parse_public_key_material(KeyId, Encoded) ->- case decode_key_material(Encoded) of- {ok, PublicKey} when byte_size(PublicKey) =:= 32 -> {ok, KeyId, PublicKey};- {ok, PublicKey} -> {error, {invalid_public_key_size, KeyId, byte_size(PublicKey)}};- {error, Reason} -> {error, {invalid_public_key, KeyId, Reason}}- end.---spec split_key_token(binary()) -> {ok, binary(), binary()} | error.-split_key_token(Token) ->- case binary:split(Token, <<":">>) of- [KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)};- _ -> split_key_token_equals(Token)- end.---spec split_key_token_equals(binary()) -> {ok, binary(), binary()} | error.-split_key_token_equals(Token) ->- case binary:split(Token, <<"=">>) of- [KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)};- _ -> error- end.---spec decode_key_material(binary()) -> {ok, binary()} | {error, term()}.-decode_key_material(Encoded) ->- case try_base64(Encoded) of- {ok, Decoded} -> {ok, Decoded};- {error, _} -> try_base64url(Encoded)- end.---spec try_base64(binary()) -> {ok, binary()} | {error, term()}.-try_base64(Encoded) ->- try- {ok, base64:decode(Encoded)}- catch- Class:Reason -> {error, {Class, Reason}}- end.---spec try_base64url(binary()) -> {ok, binary()} | {error, term()}.-try_base64url(Encoded) ->- try- {ok, base64url:decode(Encoded)}- catch- Class:Reason -> {error, {Class, Reason}}- end.---spec validate_event_hash(binary(), binary()) -> ok | {error, term()}.-validate_event_hash(Bundle, BundleHash) ->- case bundle_hash(Bundle) of- BundleHash -> ok;- Other -> {error, {bundle_hash_mismatch, Other, BundleHash}}- end.---spec event_value(atom(), map()) -> term().-event_value(Key, Event) ->- maps:get(Key, Event, maps:get(atom_to_binary(Key, utf8), Event, undefined)).---ifdef(TEST).--compress_decompress_roundtrip_test() ->- Term = #{- version => 1,- build_sha => <<"abc123">>,- modules => [#{module => <<"session_lifecycle">>, expected_current_md5 => <<0:128>>}]- },- {ok, Compressed} = compress_term(Term),- ?assert(is_binary(Compressed)),- ?assertEqual({ok, Term}, decompress_term(Compressed)).--sign_and_verify_roundtrip_test() ->- {PublicKey, PrivateKey} = ed25519_keypair(),- {ok, Compressed} = compress_term(#{version => 1}),- {ok, Signature} = sign(Compressed, PrivateKey),- Keys = #{<<"ops">> => PublicKey},- ?assertEqual(ok, verify_signature(Compressed, <<"ops">>, Signature, Keys)),- ?assertEqual(- {error, invalid_signature},- verify_signature(<<Compressed/binary, 0>>, <<"ops">>, Signature, Keys)- ).--decode_signed_event_rejects_hash_mismatch_test() ->- {PublicKey, PrivateKey} = ed25519_keypair(),- {ok, Compressed} = compress_term(#{version => 1}),- {ok, Signature} = sign(Compressed, PrivateKey),- Event = #{- signer_key_id => <<"ops">>,- signature => Signature,- bundle_sha256 => <<0:256>>,- bundle => Compressed- },- ?assertMatch(- {error, {bundle_hash_mismatch, _, _}},- decode_signed_event(Event, #{<<"ops">> => PublicKey})- ).--parse_public_keys_test() ->- PublicKey = <<1:256>>,- Encoded = base64:encode(PublicKey),- ?assertEqual(- {ok, #{<<"ops">> => PublicKey}}, parse_public_keys(<<"ops:", Encoded/binary>>)- ).--ed25519_keypair() ->- {PublicKey, PrivateKey} = crypto:generate_key(eddsa, ed25519),- {require_binary(PublicKey), require_binary(PrivateKey)}.--require_binary(Value) when is_binary(Value) ->- Value.---endif.diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erldeleted file mode 100644index df8f13c5b..000000000--- a/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl+++ /dev/null@@ -1,459 +0,0 @@-%% SPDX-License-Identifier: AGPL-3.0-or-later---module(gateway_hotpatch_cli).--typing([eqwalizer]).---export([main/1, build_bundle/2, sign_bundle/3]).---spec main([string()]) -> no_return().-main(Args) ->- halt(run_main(Args)).---spec run_main([string()]) -> non_neg_integer().-run_main(["bundle", BuildSha | ModuleOrder]) when ModuleOrder =/= [] ->- write_json_result(build_bundle(type_conv:ensure_binary(BuildSha), ModuleOrder));-run_main(["sign", SignerKeyId, PrivateKeyPath, BundlePath]) ->- write_json_result(sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath));-run_main(["append", BuildSha, CreatedBy, EventPath]) ->- Result = append_event(- type_conv:ensure_binary(BuildSha), type_conv:ensure_binary(CreatedBy), EventPath- ),- write_append_result(Result);-run_main(_Args) ->- write_usage(),- 64.---spec build_bundle(binary(), [string()]) -> {ok, map()} | {error, term()}.-build_bundle(BuildSha, ModuleSpecs) when is_binary(BuildSha), is_list(ModuleSpecs) ->- build_bundle_modules(ModuleSpecs, fun(Modules) ->- {ok, #{<<"version">> => 1, <<"build_sha">> => BuildSha, <<"modules">> => Modules}}- end).---spec sign_bundle(binary(), file:filename(), binary()) -> {ok, map()} | {error, term()}.-sign_bundle(SignerKeyId, PrivateKeyPath, BundleJson) when- is_binary(SignerKeyId), is_binary(BundleJson)-->- case decode_json(BundleJson) of- {ok, Bundle} -> sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle);- {error, Reason} -> {error, Reason}- end.---spec sign_decoded_bundle_with_key(binary(), file:filename(), term()) ->- {ok, map()} | {error, term()}.-sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle) ->- case read_private_key(PrivateKeyPath) of- {ok, PrivateKey} -> sign_decoded_bundle(SignerKeyId, Bundle, PrivateKey);- {error, Reason} -> {error, Reason}- end.---spec sign_bundle_file(string(), file:filename(), file:filename()) ->- {ok, map()} | {error, term()}.-sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath) ->- case file:read_file(BundlePath) of- {ok, BundleJson} ->- sign_bundle(type_conv:ensure_binary(SignerKeyId), PrivateKeyPath, BundleJson);- {error, Reason} ->- {error, {read_bundle_failed, Reason}}- end.---spec sign_decoded_bundle(binary(), term(), binary()) -> {ok, map()} | {error, term()}.-sign_decoded_bundle(SignerKeyId, BundleJson, PrivateKey) ->- case json_to_bundle(BundleJson) of- {ok, Bundle} -> sign_bundle_term(SignerKeyId, Bundle, PrivateKey);- {error, Reason} -> {error, Reason}- end.---spec sign_bundle_term(binary(), map(), binary()) -> {ok, map()} | {error, term()}.-sign_bundle_term(SignerKeyId, Bundle, PrivateKey) ->- case gateway_hotpatch_bundle:compress_term(Bundle) of- {ok, Compressed} -> sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey);- {error, Reason} -> {error, Reason}- end.---spec sign_compressed_bundle(binary(), binary(), binary()) -> {ok, map()} | {error, term()}.-sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey) ->- case gateway_hotpatch_bundle:sign(Compressed, PrivateKey) of- {ok, Signature} -> {ok, signed_event(SignerKeyId, Compressed, Signature)};- {error, Reason} -> {error, Reason}- end.---spec signed_event(binary(), binary(), binary()) -> map().-signed_event(SignerKeyId, Compressed, Signature) ->- #{- <<"schema_version">> => 1,- <<"kind">> => <<"beam_bundle">>,- <<"created_by">> => gateway_hotpatch_runtime:node_name(),- <<"signer_key_id">> => SignerKeyId,- <<"bundle_sha256">> => encode_bytes(gateway_hotpatch_bundle:bundle_hash(Compressed)),- <<"signature">> => encode_bytes(Signature),- <<"bundle">> => encode_bytes(Compressed)- }.---spec append_event(binary(), binary(), file:filename()) -> {ok, binary()} | {error, term()}.-append_event(BuildSha, CreatedBy, EventPath) ->- case file:read_file(EventPath) of- {ok, EventJson} -> append_event_json(BuildSha, CreatedBy, EventJson);- {error, Reason} -> {error, {read_event_failed, Reason}}- end.---spec append_event_json(binary(), binary(), binary()) -> {ok, binary()} | {error, term()}.-append_event_json(BuildSha, CreatedBy, EventJson) ->- case decode_json(EventJson) of- {ok, EventJsonTerm} -> append_event_term(BuildSha, CreatedBy, EventJsonTerm);- {error, Reason} -> {error, Reason}- end.---spec append_event_term(binary(), binary(), term()) -> {ok, binary()} | {error, term()}.-append_event_term(BuildSha, CreatedBy, EventJsonTerm) ->- case json_to_event(EventJsonTerm) of- {ok, Event} -> append_event_row(BuildSha, CreatedBy, Event);- {error, Reason} -> {error, Reason}- end.---spec build_bundle_modules([string()], fun(([map()]) -> {ok, map()})) ->- {ok, map()} | {error, term()}.-build_bundle_modules(ModuleSpecs, Fun) ->- case build_bundle_module_list(ModuleSpecs, []) of- {ok, Modules} -> Fun(Modules);- {error, Reason} -> {error, Reason}- end.---spec build_bundle_module_list([string()], [map()]) -> {ok, [map()]} | {error, term()}.-build_bundle_module_list([], Acc) ->- {ok, lists:reverse(Acc)};-build_bundle_module_list([ModuleSpec | Rest], Acc) ->- case bundle_module(ModuleSpec) of- {ok, Module} -> build_bundle_module_list(Rest, [Module | Acc]);- {error, Reason} -> {error, Reason}- end.---spec bundle_module(string()) -> {ok, map()} | {error, term()}.-bundle_module(ModuleSpec) ->- case split_module_spec(ModuleSpec) of- {ok, ModuleName, BeamPath0} -> bundle_named_module(ModuleName, BeamPath0);- {error, Reason} -> {error, Reason}- end.---spec bundle_named_module(string(), file:filename() | undefined) ->- {ok, map()} | {error, term()}.-bundle_named_module(ModuleName, BeamPath0) ->- case existing_module(ModuleName) of- {ok, Module} -> bundle_existing_module(Module, BeamPath0);- {error, Reason} -> {error, Reason}- end.---spec bundle_existing_module(atom(), file:filename() | undefined) ->- {ok, map()} | {error, term()}.-bundle_existing_module(Module, BeamPath0) ->- case collect_bundle_parts(Module, BeamPath0) of- {ok, CurrentMd5, TargetMd5, CompressedBeam} ->- {ok, build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam)};- {error, Reason} ->- {error, Reason}- end.---spec collect_bundle_parts(atom(), file:filename() | undefined) ->- {ok, binary(), binary(), binary()} | {error, term()}.-collect_bundle_parts(Module, BeamPath0) ->- case gateway_hotpatch_loader:current_md5(Module) of- {ok, CurrentMd5} -> collect_bundle_beam(Module, BeamPath0, CurrentMd5);- {error, Reason} -> {error, {current_md5_failed, Module, Reason}}- end.---spec collect_bundle_beam(atom(), file:filename() | undefined, binary()) ->- {ok, binary(), binary(), binary()} | {error, term()}.-collect_bundle_beam(Module, BeamPath0, CurrentMd5) ->- case resolve_beam_path(Module, BeamPath0) of- {ok, BeamPath} -> read_bundle_beam(Module, BeamPath, CurrentMd5);- {error, Reason} -> {error, Reason}- end.---spec read_bundle_beam(atom(), file:filename(), binary()) ->- {ok, binary(), binary(), binary()} | {error, term()}.-read_bundle_beam(Module, BeamPath, CurrentMd5) ->- case file:read_file(BeamPath) of- {ok, Beam} -> hash_bundle_beam(Module, CurrentMd5, Beam);- {error, Reason} -> {error, {read_beam_failed, Module, BeamPath, Reason}}- end.---spec hash_bundle_beam(atom(), binary(), binary()) ->- {ok, binary(), binary(), binary()} | {error, term()}.-hash_bundle_beam(Module, CurrentMd5, Beam) ->- case gateway_hotpatch_loader:beam_md5(Beam) of- {ok, TargetMd5} -> compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam);- {error, Reason} -> {error, {target_md5_failed, Module, Reason}}- end.---spec compress_bundle_beam(atom(), binary(), binary(), binary()) ->- {ok, binary(), binary(), binary()} | {error, term()}.-compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam) ->- case compress_beam(Beam) of- {ok, CompressedBeam} -> {ok, CurrentMd5, TargetMd5, CompressedBeam};- {error, Reason} -> {error, {beam_compress_failed, Module, Reason}}- end.---spec build_module_entry(atom(), binary(), binary(), binary()) -> map().-build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam) ->- #{- <<"module">> => atom_to_binary(Module, utf8),- <<"expected_current_md5">> => encode_bytes(CurrentMd5),- <<"target_md5">> => encode_bytes(TargetMd5),- <<"beam_zstd">> => encode_bytes(CompressedBeam)- }.---spec append_event_row(binary(), binary(), map()) -> {ok, binary()} | {error, term()}.-append_event_row(BuildSha, CreatedBy, Event) ->- _ = fluxer_gateway_env:load(),- case gateway_hotpatch_store:connect() of- ok ->- gateway_hotpatch_store:append_event(- BuildSha,- CreatedBy,- maps:get(signer_key_id, Event),- maps:get(signature, Event),- maps:get(bundle_sha256, Event),- maps:get(bundle, Event)- );- {error, Reason} ->- {error, Reason}- end.---spec split_module_spec(string()) ->- {ok, string(), file:filename() | undefined} | {error, term()}.-split_module_spec(ModuleSpec) ->- case string:split(ModuleSpec, "=", leading) of- [ModuleName, BeamPath] when ModuleName =/= "", BeamPath =/= "" ->- {ok, ModuleName, BeamPath};- [ModuleName] when ModuleName =/= "" ->- {ok, ModuleName, undefined};- _ ->- {error, {invalid_module_spec, ModuleSpec}}- end.---spec existing_module(string()) -> {ok, atom()} | {error, term()}.-existing_module(ModuleName) ->- try- {ok, list_to_existing_atom(ModuleName)}- catch- error:badarg -> {error, {unknown_module, ModuleName}}- end.---spec resolve_beam_path(atom(), file:filename() | undefined) ->- {ok, file:filename()} | {error, term()}.-resolve_beam_path(Module, undefined) ->- case code:which(Module) of- File when is_list(File) -> {ok, File};- Other -> {error, {module_beam_not_found, Module, Other}}- end;-resolve_beam_path(_Module, BeamPath) ->- {ok, BeamPath}.---spec compress_beam(binary()) -> {ok, binary()} | {error, term()}.-compress_beam(Beam) ->- try erlang:apply(ezstd, compress, [Beam, 3]) of- Compressed when is_binary(Compressed) -> {ok, Compressed};- {error, Reason} -> {error, Reason};- Other -> {error, Other}- catch- Class:Reason -> {error, {Class, Reason}}- end.---spec decode_json(binary()) -> {ok, term()} | {error, term()}.-decode_json(Json) ->- try- {ok, json:decode(Json)}- catch- Class:Reason -> {error, {decode_json_failed, Class, Reason}}- end.---spec read_private_key(file:filename()) -> {ok, binary()} | {error, term()}.-read_private_key(Path) ->- case file:read_file(Path) of- {ok, PrivateKey} when byte_size(PrivateKey) =:= 32 ->- {ok, PrivateKey};- {ok, PrivateKey} ->- {error, {invalid_private_key_size, byte_size(PrivateKey)}};- {error, Reason} ->- {error, {read_private_key_failed, Reason}}- end.---spec json_to_bundle(term()) -> {ok, map()} | {error, term()}.-json_to_bundle(#{- <<"version">> := Version, <<"build_sha">> := BuildSha, <<"modules">> := Modules-}) when is_list(Modules) ->- case json_to_module_entries(Modules, []) of- {ok, Entries} ->- {ok, #{- <<"version">> => Version,- <<"build_sha">> => BuildSha,- <<"modules">> => Entries- }};- {error, Reason} ->- {error, Reason}- end;-json_to_bundle(Other) ->- {error, {invalid_bundle_json, Other}}.---spec json_to_module_entries([term()], [map()]) -> {ok, [map()]} | {error, term()}.-json_to_module_entries([], Acc) ->- {ok, lists:reverse(Acc)};-json_to_module_entries([Module | Rest], Acc) ->- case json_to_module_entry(Module) of- {ok, Entry} -> json_to_module_entries(Rest, [Entry | Acc]);- {error, Reason} -> {error, Reason}- end.---spec json_to_module_entry(term()) -> {ok, map()} | {error, term()}.-json_to_module_entry(#{- <<"module">> := Module,- <<"expected_current_md5">> := ExpectedMd5,- <<"target_md5">> := TargetMd5,- <<"beam_zstd">> := BeamZstd-}) ->- decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd);-json_to_module_entry(Other) ->- {error, {invalid_module_entry_json, Other}}.---spec decode_module_entry(term(), term(), term(), term()) ->- {ok, map()} | {error, term()}.-decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) when- is_binary(Module), is_binary(ExpectedMd5), is_binary(TargetMd5), is_binary(BeamZstd)-->- case decode_bytes(ExpectedMd5) of- {ok, ExpectedMd5Bytes} ->- decode_module_entry_target(Module, ExpectedMd5Bytes, TargetMd5, BeamZstd);- {error, Reason} ->- {error, Reason}- end;-decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) ->- {error, {invalid_module_entry_json, {Module, ExpectedMd5, TargetMd5, BeamZstd}}}.---spec decode_module_entry_target(binary(), binary(), binary(), binary()) ->- {ok, map()} | {error, term()}.-decode_module_entry_target(Module, ExpectedMd5, TargetMd5, BeamZstd) ->- case decode_bytes(TargetMd5) of- {ok, TargetMd5Bytes} ->- decode_module_entry_beam(Module, ExpectedMd5, TargetMd5Bytes, BeamZstd);- {error, Reason} ->- {error, Reason}- end.---spec decode_module_entry_beam(binary(), binary(), binary(), binary()) ->- {ok, map()} | {error, term()}.-decode_module_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd) ->- case decode_bytes(BeamZstd) of- {ok, BeamZstdBytes} ->- {ok, #{- <<"module">> => Module,- <<"expected_current_md5">> => ExpectedMd5,- <<"target_md5">> => TargetMd5,- <<"beam_zstd">> => BeamZstdBytes- }};- {error, Reason} ->- {error, Reason}- end.---spec json_to_event(term()) -> {ok, map()} | {error, term()}.-json_to_event(#{- <<"signer_key_id">> := SignerKeyId,- <<"bundle_sha256">> := BundleSha256,- <<"signature">> := Signature,- <<"bundle">> := Bundle-}) ->- decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle);-json_to_event(Other) ->- {error, {invalid_event_json, Other}}.---spec decode_event_bytes(term(), term(), term(), term()) ->- {ok, map()} | {error, term()}.-decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) when- is_binary(SignerKeyId), is_binary(BundleSha256), is_binary(Signature), is_binary(Bundle)-->- case decode_bytes(BundleSha256) of- {ok, BundleSha256Bytes} ->- decode_event_signature(SignerKeyId, BundleSha256Bytes, Signature, Bundle);- {error, Reason} ->- {error, Reason}- end;-decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) ->- {error, {invalid_event_json, {SignerKeyId, BundleSha256, Signature, Bundle}}}.---spec decode_event_signature(binary(), binary(), binary(), binary()) ->- {ok, map()} | {error, term()}.-decode_event_signature(SignerKeyId, BundleSha256, Signature, Bundle) ->- case decode_bytes(Signature) of- {ok, SignatureBytes} ->- decode_event_bundle(SignerKeyId, BundleSha256, SignatureBytes, Bundle);- {error, Reason} ->- {error, Reason}- end.---spec decode_event_bundle(binary(), binary(), binary(), binary()) ->- {ok, map()} | {error, term()}.-decode_event_bundle(SignerKeyId, BundleSha256, Signature, Bundle) ->- case decode_bytes(Bundle) of- {ok, BundleBytes} ->- {ok, #{- signer_key_id => SignerKeyId,- bundle_sha256 => BundleSha256,- signature => Signature,- bundle => BundleBytes- }};- {error, Reason} ->- {error, Reason}- end.---spec encode_bytes(binary()) -> binary().-encode_bytes(Binary) ->- base64:encode(Binary).---spec decode_bytes(binary()) -> {ok, binary()} | {error, term()}.-decode_bytes(Encoded) when is_binary(Encoded) ->- try- {ok, base64:decode(Encoded)}- catch- Class:Reason -> {error, {invalid_base64_bytes, Encoded, Class, Reason}}- end.---spec write_json_result({ok, map()} | {error, term()}) -> non_neg_integer().-write_json_result({ok, Term}) ->- write_stdout("~ts~n", [json:encode(Term)]),- 0;-write_json_result({error, Reason}) ->- write_error(Reason).---spec write_append_result({ok, binary()} | {error, term()}) -> non_neg_integer().-write_append_result({ok, EventId}) ->- write_stdout("appended hotpatch event ~ts~n", [gateway_hotpatch_loader:hex(EventId)]),- 0;-write_append_result({error, Reason}) ->- write_error(Reason).---spec write_error(term()) -> non_neg_integer().-write_error(Reason) ->- write_stderr("gateway hotpatch failed: ~0tp~n", [Reason]),- 1.---spec write_usage() -> ok.-write_usage() ->- write_stderr(- "usage: gateway_hotpatch_cli bundle BUILD_SHA module_a=/path/to/module_a.beam ...~n"- " gateway_hotpatch_cli sign SIGNER_KEY_ID PRIVATE_KEY_RAW_FILE bundle.json~n"- " gateway_hotpatch_cli append BUILD_SHA CREATED_BY signed-event.json~n",- []- ).---spec write_stdout(io:format(), [term()]) -> ok.-write_stdout(Format, Args) ->- write_stream(standard_io, Format, Args).---spec write_stderr(io:format(), [term()]) -> ok.-write_stderr(Format, Args) ->- write_stream(standard_error, Format, Args).---spec write_stream(file:io_device() | standard_io | standard_error, io:format(), [term()]) ->- ok.-write_stream(Device, Format, Args) ->- Output = iolist_to_binary(io_lib:format(Format, Args)),- _ = file:write(Device, Output),- ok.diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erldeleted file mode 100644index c6b81d49d..000000000--- a/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl+++ /dev/null@@ -1,433 +0,0 @@-%% SPDX-License-Identifier: AGPL-3.0-or-later---module(gateway_hotpatch_loader).--typing([eqwalizer]).---export([- apply_bundle/1,- current_md5/1,- beam_module/1,- beam_md5/1,- hex/1-]).---ifdef(TEST).--include_lib("eunit/include/eunit.hrl").--endif.---type module_entry() :: map().--type apply_result() :: {ok, applied | skipped, atom(), binary()} | {error, term()}.---spec apply_bundle(map()) -> {ok, map()} | {error, term()}.-apply_bundle(Bundle) when is_map(Bundle) ->- case- {- entry_value(version, Bundle),- entry_value(build_sha, Bundle),- entry_value(modules, Bundle)- }- of- {1, BundleBuildSha, Modules} when is_list(Modules) ->- apply_versioned_bundle(BundleBuildSha, Modules);- _ ->- {error, invalid_bundle}- end;-apply_bundle(_Bundle) ->- {error, invalid_bundle}.---spec apply_versioned_bundle(term(), [module_entry()]) -> {ok, map()} | {error, term()}.-apply_versioned_bundle(BundleBuildSha, Modules) ->- CurrentBuildSha = gateway_hotpatch_runtime:build_sha(),- case normalize_binary(BundleBuildSha) of- CurrentBuildSha -> apply_modules(Modules, #{applied => [], skipped => []});- Other -> {error, {build_sha_mismatch, Other, CurrentBuildSha}}- end.---spec current_md5(atom()) -> {ok, binary()} | {error, term()}.-current_md5(Module) when is_atom(Module) ->- case hotpatch_loaded_md5(Module) of- {ok, Md5} -> {ok, Md5};- error -> current_md5_result(Module, current_md5_from_loaded_or_file(Module))- end.---spec current_md5_result(atom(), {ok, binary()} | {error, term()}) ->- {ok, binary()} | {error, term()}.-current_md5_result(_Module, {ok, Md5}) ->- {ok, Md5};-current_md5_result(Module, {error, Reason}) ->- fallback_current_md5(Module, Reason).---spec fallback_current_md5(atom(), term()) -> {ok, binary()} | {error, term()}.-fallback_current_md5(Module, Reason) ->- case hotpatch_loaded_md5(Module) of- {ok, Md5} -> {ok, Md5};- error -> {error, Reason}- end.---spec current_md5_from_loaded_or_file(atom()) -> {ok, binary()} | {error, term()}.-current_md5_from_loaded_or_file(Module) ->- case code:get_object_code(Module) of- {Module, Beam, _File} when is_binary(Beam) -> beam_md5(Beam);- error -> current_md5_from_file(Module)- end.---spec beam_module(binary()) -> {ok, atom()} | {error, term()}.-beam_module(Beam) when is_binary(Beam) ->- case beam_lib:info(Beam) of- Info when is_list(Info) -> beam_module_from_info(Info);- Other -> {error, {beam_info_failed, Other}}- end.---spec beam_module_from_info(list()) -> {ok, atom()} | {error, term()}.-beam_module_from_info(Info) ->- case lists:keyfind(module, 1, Info) of- {module, Module} when is_atom(Module) -> {ok, Module};- _ -> {error, module_not_found}- end.---spec beam_md5(binary()) -> {ok, binary()} | {error, term()}.-beam_md5(Beam) when is_binary(Beam) ->- case beam_lib:md5(Beam) of- {ok, {_Module, Md5}} when is_binary(Md5) -> {ok, Md5};- Error -> {error, {beam_md5_failed, Error}}- end.---spec hex(binary()) -> binary().-hex(Binary) when is_binary(Binary) ->- iolist_to_binary([[hex_nibble(High), hex_nibble(Low)] || <<High:4, Low:4>> <= Binary]).---spec apply_modules([module_entry()], map()) -> {ok, map()} | {error, term()}.-apply_modules([], Acc) ->- {ok, Acc#{module_count => length(maps:get(applied, Acc)) + length(maps:get(skipped, Acc))}};-apply_modules([Entry | Rest], Acc) ->- case apply_module(Entry) of- {ok, applied, Module, TargetMd5} ->- apply_modules(- Rest,- Acc#{applied => [{Module, hex(TargetMd5)} | maps:get(applied, Acc)]}- );- {ok, skipped, Module, TargetMd5} ->- apply_modules(- Rest,- Acc#{skipped => [{Module, hex(TargetMd5)} | maps:get(skipped, Acc)]}- );- {error, Reason} ->- {error, Reason}- end.---spec apply_module(module_entry()) -> apply_result().-apply_module(Entry) ->- with_entry(Entry, fun apply_valid_entry/4).---spec apply_valid_entry(atom(), binary(), binary(), binary()) -> apply_result().-apply_valid_entry(Module, ExpectedMd5, TargetMd5, Beam) ->- case current_md5(Module) of- {ok, TargetMd5} ->- {ok, skipped, Module, TargetMd5};- {ok, ExpectedMd5} ->- load_module(Module, Beam, TargetMd5);- {ok, CurrentMd5} ->- {error, {md5_mismatch, Module, hex(CurrentMd5), hex(ExpectedMd5)}};- {error, Reason} ->- {error, {current_md5_failed, Module, Reason}}- end.---spec with_entry(module_entry(), fun((atom(), binary(), binary(), binary()) -> apply_result())) ->- apply_result().-with_entry(Entry, Fun) when is_map(Entry), is_function(Fun, 4) ->- case normalize_module(entry_value(module, Entry)) of- {ok, Module} ->- ExpectedMd5 = entry_value(expected_current_md5, Entry),- TargetMd5 = entry_value(target_md5, Entry),- BeamZstd = entry_value(beam_zstd, Entry),- with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun);- {error, Reason} ->- {error, Reason}- end;-with_entry(_Entry, _Fun) ->- {error, invalid_module_entry}.---spec with_entry_beam(atom(), term(), term(), term(), fun(- (atom(), binary(), binary(), binary()) -> apply_result()-)) ->- apply_result().-with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun) when- is_binary(ExpectedMd5),- is_binary(TargetMd5),- byte_size(ExpectedMd5) =:= 16,- byte_size(TargetMd5) =:= 16,- is_binary(BeamZstd)-->- case decompress_beam(BeamZstd) of- {ok, Beam} -> validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun);- {error, Reason} -> {error, {beam_decompress_failed, Module, Reason}}- end;-with_entry_beam(Module, _ExpectedMd5, _TargetMd5, _BeamZstd, _Fun) ->- {error, {invalid_module_entry, Module}}.---spec validate_beam(atom(), binary(), binary(), binary(), fun(- (atom(), binary(), binary(), binary()) -> apply_result()-)) ->- apply_result().-validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun) ->- case beam_module(Beam) of- {ok, Module} -> validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun);- {ok, OtherModule} -> {error, {beam_module_mismatch, Module, OtherModule}};- {error, Reason} -> {error, {beam_module_failed, Module, Reason}}- end.---spec validate_beam_md5(atom(), binary(), binary(), binary(), fun(- (atom(), binary(), binary(), binary()) -> apply_result()-)) ->- apply_result().-validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun) ->- case beam_md5(Beam) of- {ok, TargetMd5} ->- Fun(Module, ExpectedMd5, TargetMd5, Beam);- {ok, OtherMd5} ->- {error, {target_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}};- {error, Reason} ->- {error, {target_md5_failed, Module, Reason}}- end.---spec load_module(atom(), binary(), binary()) -> apply_result().-load_module(Module, Beam, TargetMd5) ->- case code:soft_purge(Module) of- true -> load_purged_module(Module, Beam, TargetMd5);- false -> {error, {soft_purge_failed, Module}}- end.---spec load_purged_module(atom(), binary(), binary()) -> apply_result().-load_purged_module(Module, Beam, TargetMd5) ->- case code:load_binary(Module, atom_to_list(Module) ++ ".beam", Beam) of- {module, Module} ->- put_hotpatch_loaded_md5(Module, TargetMd5),- verify_loaded_module(Module, TargetMd5);- {error, Reason} ->- {error, {load_binary_failed, Module, Reason}}- end.---spec verify_loaded_module(atom(), binary()) -> apply_result().-verify_loaded_module(Module, TargetMd5) ->- case current_md5(Module) of- {ok, TargetMd5} ->- {ok, applied, Module, TargetMd5};- {ok, OtherMd5} ->- {error, {post_load_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}};- {error, Reason} ->- {error, {post_load_md5_failed, Module, Reason}}- end.---spec current_md5_from_file(atom()) -> {ok, binary()} | {error, term()}.-current_md5_from_file(Module) ->- case code:which(Module) of- File when is_list(File) -> current_md5_from_path(Module, File);- preloaded -> {error, preloaded};- non_existing -> {error, not_loaded};- Other -> {error, {not_loadable, Other}}- end.---spec current_md5_from_path(atom(), file:filename()) -> {ok, binary()} | {error, term()}.-current_md5_from_path(Module, File) ->- case beam_lib:md5(File) of- {ok, {Module, Md5}} when is_binary(Md5) -> {ok, Md5};- Error -> {error, {beam_file_md5_failed, Error}}- end.---spec hotpatch_loaded_md5(atom()) -> {ok, binary()} | error.-hotpatch_loaded_md5(Module) ->- case persistent_term:get({?MODULE, loaded_md5, Module}, undefined) of- Md5 when is_binary(Md5), byte_size(Md5) =:= 16 -> {ok, Md5};- _ -> error- end.---spec put_hotpatch_loaded_md5(atom(), binary()) -> ok.-put_hotpatch_loaded_md5(Module, Md5) ->- persistent_term:put({?MODULE, loaded_md5, Module}, Md5).---spec decompress_beam(binary()) -> {ok, binary()} | {error, term()}.-decompress_beam(Compressed) ->- try erlang:apply(ezstd, decompress, [Compressed]) of- Beam when is_binary(Beam) -> {ok, Beam};- Beam when is_list(Beam) -> {ok, iolist_to_binary(Beam)};- {error, Reason} -> {error, Reason};- Other -> {error, Other}- catch- Class:Reason -> {error, {Class, Reason}}- end.---spec normalize_module(term()) -> {ok, atom()} | {error, term()}.-normalize_module(Module) when is_atom(Module) ->- {ok, Module};-normalize_module(Module) when is_binary(Module) ->- try- {ok, binary_to_existing_atom(Module, utf8)}- catch- error:badarg -> {error, {unknown_module, Module}}- end;-normalize_module(Module) when is_list(Module) ->- normalize_module(type_conv:ensure_binary(Module));-normalize_module(Module) ->- {error, {invalid_module, Module}}.---spec normalize_binary(term()) -> binary().-normalize_binary(Bin) when is_binary(Bin) -> Bin;-normalize_binary(List) when is_list(List) -> type_conv:ensure_binary(List);-normalize_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);-normalize_binary(Other) -> term_to_binary(Other).---spec entry_value(atom(), map()) -> term().-entry_value(Key, Entry) ->- maps:get(Key, Entry, maps:get(atom_to_binary(Key, utf8), Entry, undefined)).---spec hex_nibble(0..15) -> integer().-hex_nibble(N) when N < 10 -> $0 + N;-hex_nibble(N) -> $a + (N - 10).---ifdef(TEST).--beam_module_and_md5_test() ->- {module, ?MODULE} = code:ensure_loaded(?MODULE),- {?MODULE, Binary, _File} = code:get_object_code(?MODULE),- ?assertEqual({ok, ?MODULE}, beam_module(Binary)),- {ok, Md5} = beam_md5(Binary),- ?assertEqual(16, byte_size(Md5)).--apply_bundle_rejects_build_mismatch_test() ->- Bundle = #{version => 1, build_sha => <<"definitely-not-this-build">>, modules => []},- ?assertMatch({error, {build_sha_mismatch, _, _}}, apply_bundle(Bundle)).--apply_bundle_accepts_binary_keys_test() ->- Bundle = #{- <<"version">> => 1,- <<"build_sha">> => gateway_hotpatch_runtime:build_sha(),- <<"modules">> => []- },- ?assertEqual(- {ok, #{applied => [], skipped => [], module_count => 0}}, apply_bundle(Bundle)- ).--apply_bundle_loads_new_beam_test() ->- Module = gateway_hotpatch_loader_test_target,- cleanup_test_module(Module),- try- Beam1 = compile_test_module(Module, 1),- Beam2 = compile_test_module(Module, 2),- {module, Module} = load_test_beam(Module, Beam1),- ?assertEqual(1, erlang:apply(Module, version, [])),- {ok, ExpectedMd5} = current_md5(Module),- {Entry, TargetMd5} = beam_entry(Module, ExpectedMd5, Beam2),- Bundle = #{- version => 1,- build_sha => gateway_hotpatch_runtime:build_sha(),- modules => [Entry]- },- ?assertEqual(- {ok, #{applied => [{Module, hex(TargetMd5)}], skipped => [], module_count => 1}},- apply_bundle(Bundle)- ),- ?assertEqual(2, erlang:apply(Module, version, []))- after- cleanup_test_module(Module)- end.--apply_bundle_leaves_prior_module_loaded_when_later_entry_fails_test() ->- ModuleA = gateway_hotpatch_loader_test_partial_a,- ModuleB = gateway_hotpatch_loader_test_partial_b,- cleanup_test_module(ModuleA),- cleanup_test_module(ModuleB),- try- BeamA1 = compile_test_module(ModuleA, 1),- BeamA2 = compile_test_module(ModuleA, 2),- BeamB1 = compile_test_module(ModuleB, 1),- BeamB2 = compile_test_module(ModuleB, 2),- {module, ModuleA} = load_test_beam(ModuleA, BeamA1),- {module, ModuleB} = load_test_beam(ModuleB, BeamB1),- {ok, Md5A1} = current_md5(ModuleA),- WrongExpectedMd5 = <<0:128>>,- {EntryA, _Md5A2} = beam_entry(ModuleA, Md5A1, BeamA2),- {EntryB, _Md5B2} = beam_entry(ModuleB, WrongExpectedMd5, BeamB2),- Bundle = #{- version => 1,- build_sha => gateway_hotpatch_runtime:build_sha(),- modules => [EntryA, EntryB]- },- ?assertMatch({error, {md5_mismatch, ModuleB, _, _}}, apply_bundle(Bundle)),- ?assertEqual(2, erlang:apply(ModuleA, version, [])),- ?assertEqual(1, erlang:apply(ModuleB, version, []))- after- cleanup_test_module(ModuleA),- cleanup_test_module(ModuleB)- end.--hex_test() ->- ?assertEqual(<<"0001020f10ff">>, hex(<<0, 1, 2, 15, 16, 255>>)).--compile_test_module(Module, Version) ->- Dir = test_compile_dir(),- Source = filename:join(Dir, atom_to_list(Module) ++ ".erl"),- SourceText = io_lib:format(- "-module(~p).~n-export([version/0]).~nversion() -> ~p.~n",- [Module, Version]- ),- ok = file:write_file(Source, SourceText),- case compile:file(Source, [binary, return_errors, return_warnings]) of- {ok, Module, Beam} -> Beam;- {ok, Module, Beam, _Warnings} -> Beam;- Error -> erlang:error({test_module_compile_failed, Module, Error})- end.--test_compile_dir() ->- Root =- case os:getenv("TMPDIR") of- false -> "/tmp";- Value -> Value- end,- Dir = filename:join(Root, "fluxer_hotpatch_loader_tests"),- ok = filelib:ensure_dir(filename:join(Dir, "placeholder")),- Dir.--load_test_beam(Module, Beam) ->- cleanup_test_module(Module),- BeamPath = filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam"),- ok = file:write_file(BeamPath, Beam),- code:load_abs(filename:rootname(BeamPath)).--beam_entry(Module, ExpectedMd5, TargetBeam) ->- {ok, TargetMd5} = beam_md5(TargetBeam),- {ok, BeamZstd} = compress_test_beam(TargetBeam),- {- #{- module => Module,- expected_current_md5 => ExpectedMd5,- target_md5 => TargetMd5,- beam_zstd => BeamZstd- },- TargetMd5- }.--compress_test_beam(Beam) ->- try erlang:apply(ezstd, compress, [Beam, 3]) of- Compressed when is_binary(Compressed) -> {ok, Compressed};- {error, Reason} -> {error, Reason};- Other -> {error, Other}- catch- Class:Reason -> {error, {Class, Reason}}- end.--cleanup_test_module(Module) ->- _ = code:soft_purge(Module),- _ = code:purge(Module),- _ = code:delete(Module),- _ = code:purge(Module),- erase_hotpatch_loaded_md5(Module),- _ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".erl")),- _ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam")),- ok.---spec erase_hotpatch_loaded_md5(atom()) -> ok.-erase_hotpatch_loaded_md5(Module) ->- _ = persistent_term:erase({?MODULE, loaded_md5, Module}),- ok.---endif.diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erldeleted file mode 100644index 270ccd170..000000000--- a/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl+++ /dev/null@@ -1,319 +0,0 @@-%% SPDX-License-Identifier: AGPL-3.0-or-later---module(gateway_hotpatch_reconciler).--typing([eqwalizer]).---behaviour(gen_server).---export([start_link/0, is_ready/0, status/0, reconcile_async/0]).--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).---record(state, {- enabled = false :: boolean(),- build_sha = <<"dev">> :: binary(),- public_keys = #{} :: #{binary() => binary()},- applied_event_ids = [] :: [term()],- applied_count = 0 :: non_neg_integer(),- poll_interval_ms = 5000 :: pos_integer(),- last_error = undefined :: term()-}).---type state() :: #state{}.---define(SERVER, ?MODULE).--define(POLL, poll).--define(STARTUP_RECONCILE_RETRY_MS, 1000).---spec start_link() -> gen_server:start_ret().-start_link() ->- gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).---spec is_ready() -> boolean().-is_ready() ->- gateway_hotpatch_runtime:is_ready().---spec status() -> map().-status() ->- gateway_hotpatch_runtime:status().---spec reconcile_async() -> ok.-reconcile_async() ->- gen_server:cast(?SERVER, reconcile).---spec init([]) -> {ok, state()} | {stop, term()}.-init([]) ->- case gateway_hotpatch_runtime:is_enabled() of- false -> init_disabled();- true -> init_enabled()- end.---spec handle_call(term(), {pid(), term()}, state()) -> {reply, term(), state()}.-handle_call(status, _From, State) ->- {reply, state_status(State, gateway_hotpatch_runtime:is_ready()), State};-handle_call(_Request, _From, State) ->- {reply, {error, unsupported_call}, State}.---spec handle_cast(term(), state()) -> {noreply, state()}.-handle_cast(reconcile, State) ->- {noreply, reconcile_and_publish(State)};-handle_cast(_Request, State) ->- {noreply, State}.---spec handle_info(term(), state()) -> {noreply, state()}.-handle_info(?POLL, #state{enabled = true} = State) ->- NewState = reconcile_and_publish(State),- schedule_poll(NewState),- {noreply, NewState};-handle_info(_Info, State) ->- {noreply, State}.---spec terminate(term(), state()) -> ok.-terminate(_Reason, _State) ->- ok.---spec code_change(term(), state(), term()) -> {ok, state()}.-code_change(_OldVsn, State, _Extra) ->- {ok, State}.---spec init_disabled() -> {ok, state()}.-init_disabled() ->- BuildSha = gateway_hotpatch_runtime:build_sha(),- State = #state{enabled = false, build_sha = BuildSha},- gateway_hotpatch_runtime:put_ready(true),- publish_status(State, true),- {ok, State}.---spec init_enabled() -> {ok, state()} | {stop, term()}.-init_enabled() ->- gateway_hotpatch_runtime:put_ready(false),- BuildSha = gateway_hotpatch_runtime:build_sha(),- PollIntervalMs = runtime_pos_integer(hotpatch_poll_interval_ms, 5000),- PublicKeyConfig = gateway_hotpatch_runtime:get(hotpatch_public_keys, undefined),- case gateway_hotpatch_bundle:parse_public_keys(PublicKeyConfig) of- {ok, PublicKeys} when map_size(PublicKeys) > 0 ->- State0 = #state{- enabled = true,- build_sha = BuildSha,- public_keys = PublicKeys,- poll_interval_ms = max(1000, PollIntervalMs)- },- publish_status(State0, false),- init_enabled_connected(State0);- {ok, _Empty} ->- init_enabled_config_error(BuildSha, PollIntervalMs, missing_hotpatch_public_keys);- {error, Reason} ->- init_enabled_config_error(- BuildSha, PollIntervalMs, {invalid_hotpatch_public_keys, Reason}- )- end.---spec init_enabled_config_error(binary(), pos_integer(), term()) -> {ok, state()}.-init_enabled_config_error(BuildSha, PollIntervalMs, Reason) ->- State = #state{- enabled = true,- build_sha = BuildSha,- poll_interval_ms = max(1000, PollIntervalMs),- last_error = Reason- },- gateway_hotpatch_runtime:put_ready(false),- publish_status(State, false),- logger:error("Gateway hotpatch configuration invalid: ~0tp", [Reason]),- schedule_poll(State),- {ok, State}.---spec init_enabled_connected(state()) -> {ok, state()}.-init_enabled_connected(State0) ->- finish_enabled_startup(State0).---spec finish_enabled_startup(state()) -> {ok, state()}.-finish_enabled_startup(State0) ->- TimeoutMs = runtime_pos_integer(hotpatch_startup_sync_timeout_ms, 30000),- DeadlineMs = erlang:monotonic_time(millisecond) + TimeoutMs,- finish_startup_reconcile(State0, startup_reconcile(State0, DeadlineMs)).---spec finish_startup_reconcile(state(), {ok, state()} | {error, term()}) -> {ok, state()}.-finish_startup_reconcile(_State0, {ok, State}) ->- gateway_hotpatch_runtime:put_ready(true),- publish_status(State, true),- schedule_poll(State),- {ok, State};-finish_startup_reconcile(State0, {error, Reason}) ->- Error = {hotpatch_startup_sync_failed, Reason},- State = State0#state{last_error = Error},- gateway_hotpatch_runtime:put_ready(false),- publish_status(State, false),- logger:error("Gateway hotpatch startup sync failed; keeping node unready: ~0tp", [Error]),- schedule_poll(State),- {ok, State}.---spec startup_reconcile(state(), integer()) -> {ok, state()} | {error, term()}.-startup_reconcile(State, DeadlineMs) ->- case reconcile_once(State) of- {ok, NewState} ->- {ok, NewState};- {error, Reason} ->- maybe_retry_startup_reconcile(State, DeadlineMs, Reason)- end.---spec maybe_retry_startup_reconcile(state(), integer(), term()) ->- {ok, state()} | {error, term()}.-maybe_retry_startup_reconcile(State, DeadlineMs, Reason) ->- case erlang:monotonic_time(millisecond) >= DeadlineMs of- true ->- {error, Reason};- false ->- retry_startup_reconcile(State#state{last_error = Reason}, DeadlineMs, Reason)- end.---spec retry_startup_reconcile(state(), integer(), term()) -> {ok, state()} | {error, term()}.-retry_startup_reconcile(State, DeadlineMs, Reason) ->- case gateway_retry_timer:wait_until(?STARTUP_RECONCILE_RETRY_MS, DeadlineMs) of- ok -> startup_reconcile(State, DeadlineMs);- expired -> {error, Reason};- {error, _InvalidDelay} -> {error, Reason}- end.---spec reconcile_and_publish(state()) -> state().-reconcile_and_publish(#state{enabled = false} = State) ->- gateway_hotpatch_runtime:put_ready(true),- publish_status(State, true),- State;-reconcile_and_publish(State) ->- case reconcile_once(State) of- {ok, NewState} ->- gateway_hotpatch_runtime:put_ready(true),- publish_status(NewState, true),- NewState;- {error, Reason} ->- publish_reconcile_error(State, Reason)- end.---spec reconcile_once(state()) -> {ok, state()} | {error, term()}.-reconcile_once(#state{build_sha = BuildSha} = State) ->- case gateway_hotpatch_store:connect() of- ok -> fetch_and_apply_events(BuildSha, State);- {error, Reason} -> {error, {store_connect_failed, Reason}}- end.---spec fetch_and_apply_events(binary(), state()) -> {ok, state()} | {error, term()}.-fetch_and_apply_events(BuildSha, State) ->- case gateway_hotpatch_store:fetch_events(BuildSha) of- {ok, Events} -> apply_events(Events, State);- {error, Reason} -> {error, {fetch_events_failed, Reason}}- end.---spec publish_reconcile_error(state(), term()) -> state().-publish_reconcile_error(State, {fetch_events_failed, _Reason} = Error) ->- ErrorState = State#state{last_error = Error},- Ready = gateway_hotpatch_runtime:is_ready(),- gateway_hotpatch_runtime:put_ready(Ready),- publish_status(ErrorState, Ready),- logger:warning("Gateway hotpatch fetch failed; keeping current readiness: ~0tp", [Error]),- ErrorState;-publish_reconcile_error(State, Reason) ->- ErrorState = State#state{last_error = Reason},- gateway_hotpatch_runtime:put_ready(false),- publish_status(ErrorState, false),- logger:error("Gateway hotpatch reconciliation failed: ~0tp", [Reason]),- ErrorState.---spec apply_events([map()], state()) -> {ok, state()} | {error, term()}.-apply_events([], State) ->- {ok, State#state{last_error = undefined}};-apply_events([Event | Rest], State) ->- EventId = maps:get(event_id, Event, undefined),- case lists:member(EventId, State#state.applied_event_ids) of- true ->- apply_events(Rest, State);- false ->- apply_new_event(EventId, Event, Rest, State)- end.---spec apply_new_event(term(), map(), [map()], state()) -> {ok, state()} | {error, term()}.-apply_new_event(EventId, Event, Rest, State) ->- case apply_event(Event, State) of- {ok, NewState} -> apply_events(Rest, NewState);- {error, Reason} -> {error, {event_apply_failed, EventId, Reason}}- end.---spec apply_event(map(), state()) -> {ok, state()} | {error, term()}.-apply_event(Event, #state{public_keys = PublicKeys, build_sha = BuildSha} = State) ->- EventId = maps:get(event_id, Event, undefined),- BundleHash = maps:get(bundle_sha256, Event, <<>>),- _ = code:ensure_loaded(gateway_hotpatch_loader),- case gateway_hotpatch_bundle:decode_signed_event(Event, PublicKeys) of- {ok, Bundle} ->- apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State);- {error, Reason} ->- audit_event(BuildSha, EventId, #{bundle_sha256 => BundleHash}, {error, Reason}),- {error, Reason}- end.---spec apply_decoded_event(binary(), term(), binary(), map(), state()) ->- {ok, state()} | {error, term()}.-apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State) ->- Summary0 = #{bundle_sha256 => BundleHash},- case gateway_hotpatch_loader:apply_bundle(Bundle) of- {ok, Summary} ->- handle_applied_event(BuildSha, EventId, Summary0, Summary, State);- {error, Reason} ->- audit_event(BuildSha, EventId, Summary0, {error, Reason}),- {error, Reason}- end.---spec handle_applied_event(binary(), term(), map(), map(), state()) -> {ok, state()}.-handle_applied_event(BuildSha, EventId, Summary0, Summary, State) ->- AuditSummary = maps:merge(Summary0, Summary),- audit_event(BuildSha, EventId, AuditSummary, ok),- logger:notice("Applied gateway hotpatch event ~0tp summary=~0tp", [EventId, AuditSummary]),- {ok, State#state{- applied_event_ids = [EventId | State#state.applied_event_ids],- applied_count = State#state.applied_count + 1,- last_error = undefined- }}.---spec audit_event(binary(), term(), map(), ok | {error, term()}) -> ok.-audit_event(BuildSha, EventId, Summary, Result) when is_binary(EventId) ->- case- gateway_hotpatch_store:audit_applied(- BuildSha, gateway_hotpatch_runtime:node_name(), EventId, Summary, Result- )- of- ok ->- ok;- {error, Reason} ->- logger:warning("Gateway hotpatch audit write failed: ~0tp", [Reason]),- ok- end;-audit_event(_BuildSha, _EventId, _Summary, _Result) ->- ok.---spec publish_status(state(), boolean()) -> ok.-publish_status(State, Ready) ->- gateway_hotpatch_runtime:put_status(state_status(State, Ready)).---spec state_status(state(), boolean()) -> map().-state_status(State, Ready) ->- #{- enabled => State#state.enabled,- ready => Ready,- build_sha => State#state.build_sha,- applied_count => State#state.applied_count,- applied_event_count => length(State#state.applied_event_ids),- last_error => State#state.last_error- }.---spec schedule_poll(state()) -> ok.-schedule_poll(#state{poll_interval_ms = PollIntervalMs}) ->- erlang:send_after(PollIntervalMs, self(), ?POLL),- ok.---spec runtime_pos_integer(atom(), pos_integer()) -> pos_integer().-runtime_pos_integer(Key, Default) ->- normalize_pos_integer(gateway_hotpatch_runtime:get(Key, Default), Default).---spec normalize_pos_integer(term(), pos_integer()) -> pos_integer().-normalize_pos_integer(Value, _Default) when is_integer(Value), Value > 0 ->- Value;-normalize_pos_integer(_Value, Default) ->- Default.diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erldeleted file mode 100644index d878d7b17..000000000--- a/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl+++ /dev/null@@ -1,78 +0,0 @@-%% SPDX-License-Identifier: AGPL-3.0-or-later---module(gateway_hotpatch_runtime).--typing([eqwalizer]).---export([- build_sha/0,- node_name/0,- is_enabled/0,- get/2,- put_ready/1,- is_ready/0,- put_status/1,- status/0-]).---define(READY_KEY, {gateway_hotpatch, ready}).--define(STATUS_KEY, {gateway_hotpatch, status}).---spec build_sha() -> binary().-build_sha() ->- case os:getenv("BUILD_SHA") of- false -> build_version();- "" -> build_version();- Value -> type_conv:ensure_binary(Value, <<"dev">>)- end.---spec build_version() -> binary().-build_version() ->- case os:getenv("BUILD_VERSION") of- false -> <<"dev">>;- "" -> <<"dev">>;- Value -> type_conv:ensure_binary(Value, <<"dev">>)- end.---spec node_name() -> binary().-node_name() ->- atom_to_binary(node(), utf8).---spec is_enabled() -> boolean().-is_enabled() ->- case fluxer_gateway_env:get(hotpatch_enabled) of- true -> true;- _ -> false- end.---spec get(atom(), term()) -> term().-get(Key, Default) ->- case fluxer_gateway_env:get(Key) of- undefined -> Default;- Value -> Value- end.---spec put_ready(boolean()) -> ok.-put_ready(Ready) when is_boolean(Ready) ->- persistent_term:put(?READY_KEY, Ready),- ok.---spec is_ready() -> boolean().-is_ready() ->- case persistent_term:get(?READY_KEY, true) of- false -> false;- _ -> true- end.---spec put_status(map()) -> ok.-put_status(Status) ->- persistent_term:put(?STATUS_KEY, Status),- ok.---spec status() -> map().-status() ->- persistent_term:get(?STATUS_KEY, #{- enabled => false,- ready => true,- build_sha => build_sha(),- applied_count => 0- }).diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_store.erldeleted file mode 100644index 7759c70de..000000000--- a/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl+++ /dev/null@@ -1,277 +0,0 @@-%% SPDX-License-Identifier: AGPL-3.0-or-later---module(gateway_hotpatch_store).--typing([eqwalizer]).---export([- connect/0,- fetch_events/1,- append_event/5,- append_event/6,- audit_applied/5,- normalize_event_rows/1-]).---ifdef(TEST).--include_lib("eunit/include/eunit.hrl").--endif.---define(STMT_FETCH_EVENTS, gateway_hotpatch_fetch_events_by_build).--define(STMT_APPEND_EVENT, gateway_hotpatch_append_event).--define(STMT_AUDIT_APPLIED, gateway_hotpatch_audit_applied).---spec connect() -> ok | {error, term()}.-connect() ->- case gateway_hotpatch_runtime:get(hotpatch_cassandra_hosts, undefined) of- Hosts when is_list(Hosts), Hosts =/= [] -> connect_hosts(Hosts);- Hosts when is_binary(Hosts), byte_size(Hosts) > 0 ->- start_erlcass(binary_to_list(Hosts));- _ ->- {error, missing_hotpatch_cassandra_hosts}- end.---spec connect_hosts(term()) -> ok | {error, term()}.-connect_hosts(Hosts) ->- case type_conv:to_list(Hosts) of- String when is_list(String), String =/= [] -> start_erlcass(String);- _ -> {error, missing_hotpatch_cassandra_hosts}- end.---spec fetch_events(binary()) -> {ok, [map()]} | {error, term()}.-fetch_events(BuildSha) when is_binary(BuildSha) ->- case erlcass:execute(?STMT_FETCH_EVENTS, [BuildSha]) of- {ok, _Columns, Rows} -> {ok, normalize_event_rows(Rows)};- ok -> {ok, []};- {error, Reason} -> {error, Reason}- end.---spec append_event(binary(), binary(), binary(), binary(), binary()) ->- {ok, binary()} | {error, term()}.-append_event(CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when- is_binary(CreatedBy),- is_binary(SignerKeyId),- is_binary(Signature),- is_binary(BundleSha256),- is_binary(Bundle)-->- BuildSha = gateway_hotpatch_runtime:build_sha(),- append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle).---spec append_event(binary(), binary(), binary(), binary(), binary(), binary()) ->- {ok, binary()} | {error, term()}.-append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when- is_binary(BuildSha),- is_binary(CreatedBy),- is_binary(SignerKeyId),- is_binary(Signature),- is_binary(BundleSha256),- is_binary(Bundle)-->- case erlcass_uuid:gen_time() of- {ok, EventId} ->- Params = append_event_params(- BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle- ),- execute_append_event(EventId, Params);- {error, Reason} ->- {error, {event_id_failed, Reason}}- end.---spec append_event_params(- binary(), binary(), binary(), binary(), binary(), binary(), binary()-) -> list().-append_event_params(BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle) ->- [- BuildSha,- EventId,- 1,- <<"beam_bundle">>,- CreatedBy,- SignerKeyId,- BundleSha256,- Signature,- Bundle- ].---spec execute_append_event(binary(), list()) -> {ok, binary()} | {error, term()}.-execute_append_event(EventId, Params) ->- case erlcass:execute(?STMT_APPEND_EVENT, Params) of- ok -> {ok, EventId};- {ok, _Columns, _Rows} -> {ok, EventId};- {error, Reason} -> {error, Reason}- end.---spec audit_applied(binary(), binary(), binary(), map(), ok | {error, term()}) ->- ok | {error, term()}.-audit_applied(BuildSha, NodeName, EventId, Summary, Result) when- is_binary(BuildSha), is_binary(NodeName), is_binary(EventId), is_map(Summary)-->- ModuleCount = maps:get(module_count, Summary, 0),- BundleSha256 = maps:get(bundle_sha256, Summary, <<>>),- Status = audit_status(Result),- Error = audit_error(Result),- Params = [BuildSha, NodeName, EventId, ModuleCount, BundleSha256, Status, Error],- case erlcass:execute(?STMT_AUDIT_APPLIED, Params) of- ok -> ok;- {ok, _Columns, _Rows} -> ok;- {error, Reason} -> {error, Reason}- end.---spec normalize_event_rows([list()]) -> [map()].-normalize_event_rows(Rows) when is_list(Rows) ->- lists:filtermap(fun normalize_event_row/1, Rows).---spec start_erlcass(string()) -> ok | {error, term()}.-start_erlcass(Hosts) ->- Keyspace = gateway_hotpatch_runtime:get(hotpatch_cassandra_keyspace, <<"fluxer">>),- Port = gateway_hotpatch_runtime:get(hotpatch_cassandra_port, 9042),- Options0 = [- {contact_points, type_conv:ensure_binary(Hosts)},- {port, Port},- {latency_aware_routing, true},- {token_aware_routing, true},- {tcp_nodelay, true},- {tcp_keepalive, {true, 60}},- {connect_timeout, 5000},- {request_timeout, 5000},- {retry_policy, {default, true}},- {default_consistency_level, 6}- ],- case maybe_credentials(Options0) of- {ok, Options} -> start_erlcass_with_options(Keyspace, Options);- {error, Reason} -> {error, Reason}- end.---spec start_erlcass_with_options(binary(), list()) -> ok | {error, term()}.-start_erlcass_with_options(Keyspace, Options) ->- application:set_env(erlcass, keyspace, Keyspace),- application:set_env(erlcass, cluster_options, Options),- application:set_env(erlcass, log_level, 2),- case application:ensure_all_started(erlcass) of- {ok, _Apps} -> prepare_statements();- {error, {erlcass, {already_started, erlcass}}} -> prepare_statements();- {error, Reason} -> {error, {erlcass_start_failed, Reason}}- end.---spec prepare_statements() -> ok | {error, term()}.-prepare_statements() ->- Statements = [- {?STMT_FETCH_EVENTS, <<- "SELECT event_id, schema_version, kind, created_by, signer_key_id, "- "bundle_sha256, signature, bundle "- "FROM gateway_hotpatch_events_by_build WHERE build_sha = ?"- >>},- {?STMT_APPEND_EVENT, <<- "INSERT INTO gateway_hotpatch_events_by_build "- "(build_sha, event_id, schema_version, kind, created_at, created_by, "- "signer_key_id, bundle_sha256, signature, bundle) "- "VALUES (?, ?, ?, ?, toTimestamp(now()), ?, ?, ?, ?, ?)"- >>},- {?STMT_AUDIT_APPLIED, <<- "INSERT INTO gateway_hotpatch_applied_by_node "- "(build_sha, node_name, event_id, applied_at, module_count, bundle_sha256, "- "status, error) "- "VALUES (?, ?, ?, toTimestamp(now()), ?, ?, ?, ?)"- >>}- ],- prepare_statements(Statements).---spec prepare_statements([{atom(), binary()}]) -> ok | {error, term()}.-prepare_statements([]) ->- ok;-prepare_statements([{Id, Query} | Rest]) ->- case prepare_statement(Id, Query) of- ok -> prepare_statements(Rest);- {error, Reason} -> {error, Reason}- end.---spec prepare_statement(atom(), binary()) -> ok | {error, term()}.-prepare_statement(Id, Query) ->- case erlcass:add_prepare_statement(Id, Query) of- ok -> ok;- {error, already_exist} -> ok;- {error, Reason} -> {error, {prepare_failed, Id, Reason}}- end.---spec maybe_credentials(list()) -> {ok, list()} | {error, term()}.-maybe_credentials(Options) ->- Username = gateway_hotpatch_runtime:get(hotpatch_cassandra_username, undefined),- Password = gateway_hotpatch_runtime:get(hotpatch_cassandra_password, undefined),- case {Username, Password} of- {U, P} when is_binary(U), byte_size(U) > 0, is_binary(P), byte_size(P) > 0 ->- {ok, [{credentials, {U, P}} | Options]};- _ ->- {error, missing_hotpatch_cassandra_credentials}- end.---spec normalize_event_row(list()) -> {true, map()} | false.-normalize_event_row([- EventId,- SchemaVersion,- Kind,- CreatedBy,- SignerKeyId,- BundleSha256,- Signature,- Bundle-]) when- is_binary(EventId),- is_binary(SignerKeyId),- is_binary(BundleSha256),- is_binary(Signature),- is_binary(Bundle)-->- {true, #{- event_id => EventId,- schema_version => SchemaVersion,- kind => Kind,- created_by => CreatedBy,- signer_key_id => SignerKeyId,- bundle_sha256 => BundleSha256,- signature => Signature,- bundle => Bundle- }};-normalize_event_row(_Row) ->- false.---spec audit_status(ok | {error, term()}) -> binary().-audit_status(ok) -> <<"ok">>;-audit_status({error, _Reason}) -> <<"error">>.---spec audit_error(ok | {error, term()}) -> binary().-audit_error(ok) ->- <<>>;-audit_error({error, Reason}) ->- iolist_to_binary(io_lib:format("~0tp", [Reason])).---ifdef(TEST).--normalize_event_rows_test() ->- EventId = <<1:128>>,- Row = [- EventId,- 1,- <<"beam_bundle">>,- <<"hampus">>,- <<"ops">>,- <<2:256>>,- <<3:512>>,- <<"bundle">>- ],- ?assertEqual(- [- #{- event_id => EventId,- schema_version => 1,- kind => <<"beam_bundle">>,- created_by => <<"hampus">>,- signer_key_id => <<"ops">>,- bundle_sha256 => <<2:256>>,- signature => <<3:512>>,- bundle => <<"bundle">>- }- ],- normalize_event_rows([Row, [invalid]])- ).---endif.diff --git a/fluxer_gateway/src/gateway/gateway_node_router.erl b/fluxer_gateway/src/gateway/gateway_node_router.erlindex a7c2cf1f5..b2374c9f3 100644--- a/fluxer_gateway/src/gateway/gateway_node_router.erl+++ b/fluxer_gateway/src/gateway/gateway_node_router.erl@@ -98,7 +98,7 @@ fallback_owner_for_role(Role) ->-spec is_ready() -> boolean().is_ready() ->- not is_draining() andalso gateway_hotpatch_reconciler:is_ready().+ not is_draining().-spec is_draining() -> boolean().is_draining() ->diff --git a/fluxer_gateway/src/gateway/metrics_handler.erl b/fluxer_gateway/src/gateway/metrics_handler.erlindex 2c5f3bb4e..12300c1c5 100644--- a/fluxer_gateway/src/gateway/metrics_handler.erl+++ b/fluxer_gateway/src/gateway/metrics_handler.erl@@ -23,7 +23,6 @@ render_metrics() ->render_cluster_counters(),render_process_counts(),render_push_dispatcher_stats(),- render_hotpatch_metrics(),render_vm_metrics()].@@ -256,33 +255,6 @@ render_push_dispatcher_stats() ->]end.--spec render_hotpatch_metrics() -> iolist().-render_hotpatch_metrics() ->- Status = safe_apply_map(fun gateway_hotpatch_reconciler:status/0),- Ready = maps:get(ready, Status, true),- Enabled = maps:get(enabled, Status, false),- AppliedCount = maps:get(applied_count, Status, 0),- [- format_metric(- <<"fluxer_gateway_hotpatch_enabled">>,- <<"gauge">>,- <<"Gateway hotpatch reconciler enabled">>,- bool_metric(Enabled)- ),- format_metric(- <<"fluxer_gateway_hotpatch_ready">>,- <<"gauge">>,- <<"Gateway hotpatch reconciliation readiness">>,- bool_metric(Ready)- ),- format_metric(- <<"fluxer_gateway_hotpatch_applied_events_total">>,- <<"counter">>,- <<"Gateway hotpatch events applied on this node">>,- integer_to_binary(AppliedCount)- )- ].--spec render_vm_metrics() -> iolist().render_vm_metrics() ->Memory = safe_apply_list(fun erlang:memory/0),@@ -376,7 +348,3 @@ format_labeled_series(Name, Type, Help, LabelValues) ->|| {Label, Value} <- LabelValues]].---spec bool_metric(term()) -> binary().-bool_metric(true) -> <<"1">>;-bool_metric(_) -> <<"0">>.diff --git a/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl b/fluxer_gateway/src/guild/guild_member_list_sync_batch.erlindex 1b9c46615..ee09298c9 100644--- a/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl+++ b/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl@@ -159,7 +159,7 @@ flush_pending_syncs_clears_empty_batch_test() ->State = #{?SYNC_BATCH_STATE_KEY => #{pending_list_ids => #{}}},?assertEqual(#{}, flush_pending_syncs(State)).-queue_list_sync_drains_pre_hotpatch_pending_batch_test() ->+queue_list_sync_drains_preexisting_pending_batch_test() ->Ref = erlang:send_after(60000, self(), ?FLUSH_SYNC_BATCH_MSG),State = #{?SYNC_BATCH_STATE_KEY => #{diff --git a/fluxer_svc/src/postgres.rs b/fluxer_svc/src/postgres.rsindex 391f7410c..f4ea1dd7c 100644--- a/fluxer_svc/src/postgres.rs+++ b/fluxer_svc/src/postgres.rs@@ -11,6 +11,9 @@ use std::str::FromStr;use tokio_postgres::{Config as PgConfig, Row, config::SslMode, types::ToSql};use tokio_postgres_rustls::MakeRustlsConnect;+const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE: i32 = 0x4658_4b56;+const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT: &str = "120s";+#[derive(Clone, Debug)]pub struct PostgresConfig {pub url: Option<String>,@@ -134,8 +137,30 @@ pub async fn ensure_kv_schema(pool: &Pool, kv_table: &str) -> anyhow::Result<()>let messages_message_index = quote_identifier(&format!("{kv_table}_messages_message_idx"))?;let message_reactions_message_index =quote_identifier(&format!("{kv_table}_message_reactions_message_idx"))?;- let client = pool.get().await?;- client+ let mut client = pool.get().await?;+ let transaction = client+ .transaction()+ .await+ .context("failed to begin Postgres KV schema transaction")?;+ transaction+ .query_one(+ "SELECT set_config('statement_timeout', $1, true)",+ &[&POSTGRES_KV_SCHEMA_LOCK_TIMEOUT],+ )+ .await+ .context("failed to configure Postgres KV schema lock timeout")?;+ transaction+ .query_one(+ "SELECT pg_advisory_xact_lock($1, hashtext($2))",+ &[&POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, &kv_table],+ )+ .await+ .context("failed to acquire Postgres KV schema lock")?;+ transaction+ .query_one("SELECT set_config('statement_timeout', '0', true)", &[])+ .await+ .context("failed to clear Postgres KV schema lock timeout")?;+ transaction.batch_execute(&format!(r#"CREATE TABLE IF NOT EXISTS {table} (@@ -162,6 +187,10 @@ DROP INDEX IF EXISTS {old_partition_index};)).await.context("failed to ensure Postgres KV schema")?;+ transaction+ .commit()+ .await+ .context("failed to commit Postgres KV schema transaction")?;Ok(())}diff --git a/package.json b/package.jsonindex 1622ff3a5..7e282a45f 100644--- a/package.json+++ b/package.json@@ -13,11 +13,10 @@"dev:desktop:canary": "cargo run -p fluxer-dev -- desktop canary","dev:desktop:package": "cargo run -p fluxer-dev -- desktop package","dev:desktop:run": "cargo run -p fluxer-dev -- desktop run",- "dev:infra:down": "docker compose -f .devcontainer/docker-compose.yml down",- "dev:infra:restart": "pnpm dev:infra:down && pnpm dev:infra:up",- "dev:infra:up": "pnpm dev:infra:up:lite",- "dev:infra:up:full": "docker compose -f .devcontainer/docker-compose.yml up -d postgres cassandra valkey nats livekit elasticsearch meilisearch mailpit",- "dev:infra:up:lite": "docker compose -f .devcontainer/docker-compose.yml up -d postgres valkey nats livekit meilisearch mailpit",+ "dev:infra:restart": "pnpm dev:infra:stop && pnpm dev:infra:start",+ "dev:infra:start": "cargo run -p fluxer-dev -- infra start",+ "dev:infra:status": "cargo run -p fluxer-dev -- infra status",+ "dev:infra:stop": "cargo run -p fluxer-dev -- infra stop","dev:services": "cargo run -p fluxer-dev -- rust-services","dev:tunnel": "cargo run -p fluxer-dev -- dev --cloudflare-tunnel","dev:tunnel:configure": "cargo run -p fluxer-dev -- tunnel configure",diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.tsindex 6c7e5df55..e2165d47f 100644--- a/packages/config/src/ConfigLoader.ts+++ b/packages/config/src/ConfigLoader.ts@@ -195,6 +195,7 @@ function defaultConfig(): MasterConfig {api_key: '',api_secret: '',url: '',+ internal_url: '',webhook_url: '',},search: {diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.tsindex 28c719942..629e55c19 100644--- a/packages/config/src/MasterConfig.ts+++ b/packages/config/src/MasterConfig.ts@@ -243,6 +243,7 @@ export interface MasterConfig {api_key: string;api_secret: string;url: string;+ internal_url: string;webhook_url: string;default_region?: {id: string;diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.tsindex adb9175ce..dae1be5af 100644--- a/packages/config/src/config_loader/EnvironmentOverrides.ts+++ b/packages/config/src/config_loader/EnvironmentOverrides.ts@@ -266,6 +266,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {FLUXER_LIVEKIT_API_KEY: {path: ['integrations', 'voice', 'api_key']},FLUXER_LIVEKIT_API_SECRET: {path: ['integrations', 'voice', 'api_secret']},FLUXER_LIVEKIT_URL: {path: ['integrations', 'voice', 'url']},+ FLUXER_LIVEKIT_INTERNAL_URL: {path: ['integrations', 'voice', 'internal_url']},FLUXER_LIVEKIT_WEBHOOK_URL: {path: ['integrations', 'voice', 'webhook_url']},FLUXER_LIVEKIT_DEFAULT_REGION: {path: ['integrations', 'voice', 'default_region'], parse: parseEnvValue},FLUXER_SEARCH_ENGINE: {path: ['integrations', 'search', 'engine']},diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 83c0a9173..97cb49d5f 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -7701,8 +7701,8 @@ packages:resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==}engines: {node: '>=16'}- resolution: {integrity: sha512-DDdwPGz99nmIEv216hKSgLD+D4ikHQHjBC/seF98N9CPqRX4M5mSxT9eTV6oyisnJcuzxtZy4n17yKKQYmYQOA==}+ resolution: {integrity: sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==}resolution: {integrity: sha512-HritfMGq9V7SuESeSodHvArs0mLuMk7uh+7hQK2lqdvXrvm50aWxb4RPxkK3mPDdsgHjJ427xNRFITMH2ei+Sw==}@@ -17551,7 +17551,7 @@ snapshots:dependencies:browserslist: 4.28.1- caniuse-lite: 1.0.30001774+ caniuse-lite: 1.0.30001810fraction.js: 5.3.4picocolors: 1.1.1postcss: 8.5.6@@ -17666,7 +17666,7 @@ snapshots:dependencies:baseline-browser-mapping: 2.10.0- caniuse-lite: 1.0.30001774+ caniuse-lite: 1.0.30001810electron-to-chromium: 1.5.302node-releases: 2.0.27update-browserslist-db: 1.2.3([email protected])@@ -17761,7 +17761,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}dependencies:diff --git a/tools/ci/src/app_dev_server.rs b/tools/ci/src/app_dev_server.rsindex c63868325..cbf5d92c5 100644--- a/tools/ci/src/app_dev_server.rs+++ b/tools/ci/src/app_dev_server.rs@@ -4,9 +4,11 @@ use crate::app_wasm::resolve_app_dir;use anyhow::{Context, Result, bail, ensure};use clap::Args;use serde::{Deserialize, Serialize};+use sha2::{Digest, Sha256};use std::collections::{BTreeMap, BTreeSet};use std::ffi::OsStr;use std::fs;+use std::io::Read;use std::path::{Path, PathBuf};use std::process::Stdio;use std::time::{Duration, SystemTime, UNIX_EPOCH};@@ -23,11 +25,51 @@ pub struct AppDevServerArgs {app_dir: Option<PathBuf>,}+#[derive(Clone, Debug, Deserialize, Serialize)]+#[serde(untagged)]+enum InputFingerprint {+ Sha256 {+ sha256: String,+ #[serde(skip)]+ legacy_timestamp_ms: Option<f64>,+ },+ LegacyTimestamp(f64),+}++impl InputFingerprint {+ fn matches(&self, current: &Self) -> bool {+ match (self, current) {+ (+ Self::Sha256 { sha256, .. },+ Self::Sha256 {+ sha256: current, ..+ },+ ) => sha256 == current,+ (+ Self::LegacyTimestamp(timestamp),+ Self::Sha256 {+ legacy_timestamp_ms: Some(current),+ ..+ },+ ) => timestamp == current,+ (Self::LegacyTimestamp(timestamp), Self::LegacyTimestamp(current)) => {+ timestamp == current+ }+ _ => false,+ }+ }++ fn is_legacy(&self) -> bool {+ matches!(self, Self::LegacyTimestamp(_))+ }+}++type StepInputs = BTreeMap<String, InputFingerprint>;+#[derive(Clone, Debug, Default, Deserialize, Serialize)]#[serde(rename_all = "camelCase")]struct StepMetadata {- last_run: f64,- inputs: BTreeMap<String, f64>,+ inputs: StepInputs,}type Metadata = BTreeMap<String, StepMetadata>;@@ -103,20 +145,14 @@ impl AppDevServer {).await?;- if env_truthy("FLUXER_APP_SKIP_I18N_COMPILE") {- eprintln!("Skipping pnpm lingui:compile because FLUXER_APP_SKIP_I18N_COMPILE is set.");- } else {- self.run_cached_step(- "lingui",- gather_lingui_inputs,- "pnpm lingui:compile",- |server, shutdown| {- Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown))- },- &mut shutdown_rx,- )- .await?;- }+ self.run_cached_step(+ "lingui",+ gather_lingui_inputs,+ "pnpm lingui:compile",+ |server, shutdown| Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown)),+ &mut shutdown_rx,+ )+ .await?;if *shutdown_rx.borrow() {return Ok(());@@ -175,35 +211,50 @@ impl AppDevServer {shutdown: &mut watch::Receiver<bool>,) -> Result<()>where- G: Fn(&Path) -> Result<BTreeMap<String, f64>>,+ G: Fn(&Path) -> Result<StepInputs>,E: for<'a> FnOnce(&'a AppDevServer,&'a mut watch::Receiver<bool>,)-> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>>,{+ if *shutdown.borrow() {+ return Ok(());+ }let inputs = gather_inputs(&self.project_root)?;if !self.should_run_step(step_name, &inputs) {+ let needs_upgrade = self+ .metadata+ .get(step_name)+ .is_some_and(|entry| entry.inputs.values().any(InputFingerprint::is_legacy));+ if needs_upgrade {+ self.metadata+ .insert(step_name.to_string(), StepMetadata { inputs });+ self.save_metadata()?;+ }println!("Skipping {label} (no changes detected)");return Ok(());}execute(self, shutdown).await?;- self.metadata.insert(- step_name.to_string(),- StepMetadata {- last_run: timestamp_ms(SystemTime::now())?,- inputs,- },- );+ if *shutdown.borrow() {+ return Ok(());+ }+ self.metadata+ .insert(step_name.to_string(), StepMetadata { inputs });self.save_metadata()}- fn should_run_step(&self, step_name: &str, inputs: &BTreeMap<String, f64>) -> bool {+ fn should_run_step(&self, step_name: &str, inputs: &StepInputs) -> bool {let Some(entry) = self.metadata.get(step_name) else {return true;};- &entry.inputs != inputs+ entry.inputs.len() != inputs.len()+ || entry.inputs.iter().any(|(path, cached)| {+ inputs+ .get(path)+ .is_none_or(|current| !cached.matches(current))+ })}async fn run_command(@@ -286,7 +337,7 @@ impl AppDevServer {}}-fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap<String, f64>> {+fn collect_file_digests(project_root: &Path, paths: &[PathBuf]) -> Result<StepInputs> {let mut result = BTreeMap::new();for rel_path in paths {let absolute_path = project_root.join(rel_path);@@ -297,16 +348,16 @@ fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap"Expected {} to be a file when collecting dev server cache inputs.",rel_path.display());- result.insert(rel_path_key(rel_path), timestamp_ms(metadata.modified()?)?);+ result.insert(rel_path_key(rel_path), fingerprint_file(&absolute_path)?);}Ok(result)}-fn collect_directory_stats<P>(+fn collect_directory_digests<P>(project_root: &Path,root_rel: &Path,predicate: P,-) -> Result<BTreeMap<String, f64>>+) -> Result<StepInputs>whereP: Fn(&str) -> bool,{@@ -335,20 +386,47 @@ whereif !predicate(&key) {continue;}- result.insert(key, timestamp_ms(entry.metadata()?.modified()?)?);+ result.insert(key, fingerprint_file(entry.path())?);}Ok(result)}+fn hash_file(path: &Path) -> Result<String> {+ let mut file = fs::File::open(path)+ .with_context(|| format!("Failed to open {} for hashing", path.display()))?;+ let mut hasher = Sha256::new();+ let mut buffer = [0_u8; 64 * 1024];+ loop {+ let bytes_read = file+ .read(&mut buffer)+ .with_context(|| format!("Failed to read {} for hashing", path.display()))?;+ if bytes_read == 0 {+ break;+ }+ hasher.update(&buffer[..bytes_read]);+ }+ Ok(hex::encode(hasher.finalize()))+}++fn fingerprint_file(path: &Path) -> Result<InputFingerprint> {+ let sha256 = hash_file(path)?;+ let metadata = fs::metadata(path)+ .with_context(|| format!("Failed to stat {} after hashing", path.display()))?;+ Ok(InputFingerprint::Sha256 {+ sha256,+ legacy_timestamp_ms: Some(timestamp_ms(metadata.modified()?)?),+ })+}+fn should_walk_entry(path: &Path, skip_dirs: &BTreeSet<&str>) -> bool {path.file_name().and_then(OsStr::to_str).is_none_or(|name| !skip_dirs.contains(name))}-fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {+fn gather_wasm_inputs(project_root: &Path) -> Result<StepInputs> {let markdown_parser_rust_dir = PathBuf::from("../packages/markdown_parser/rust");- let mut inputs = collect_file_stats(+ let mut inputs = collect_file_digests(project_root,&[PathBuf::from("../tools/ci/Cargo.toml"),@@ -361,12 +439,12 @@ fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {markdown_parser_rust_dir.join("Cargo.toml"),],)?;- inputs.extend(collect_directory_stats(+ inputs.extend(collect_directory_digests(project_root,Path::new("rust/libfluxcore"),|path| !path.contains("/target/"),)?);- inputs.extend(collect_directory_stats(+ inputs.extend(collect_directory_digests(project_root,&markdown_parser_rust_dir,|path| !path.contains("/target/"),@@ -374,15 +452,15 @@ fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {Ok(inputs)}-fn gather_color_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {- collect_file_stats(+fn gather_color_inputs(project_root: &Path) -> Result<StepInputs> {+ collect_file_digests(project_root,&[PathBuf::from("scripts/GenerateColorSystem.ts")],)}-fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {- collect_file_stats(+fn gather_message_layout_inputs(project_root: &Path) -> Result<StepInputs> {+ collect_file_digests(project_root,&[PathBuf::from("scripts/GenerateMessageLayoutCss.ts"),@@ -391,8 +469,8 @@ fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String,)}-fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {- collect_file_stats(+fn gather_mask_inputs(project_root: &Path) -> Result<StepInputs> {+ collect_file_digests(project_root,&[PathBuf::from("scripts/GenerateAvatarMasks.ts"),@@ -401,14 +479,14 @@ fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {)}-fn gather_css_module_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {- collect_directory_stats(project_root, Path::new("src"), |path| {+fn gather_css_module_inputs(project_root: &Path) -> Result<StepInputs> {+ collect_directory_digests(project_root, Path::new("src"), |path| {path.ends_with(".module.css")})}-fn gather_lingui_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {- collect_directory_stats(+fn gather_lingui_inputs(project_root: &Path) -> Result<StepInputs> {+ collect_directory_digests(project_root,Path::new("src/features/i18n/locales"),|path| path.ends_with(".po"),@@ -493,12 +571,6 @@ fn timestamp_ms(timestamp: SystemTime) -> Result<f64> {* 1000.0)}-fn env_truthy(name: &str) -> bool {- std::env::var(name)- .ok()- .is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true"))-}-fn display_command(command: &str, args: &[&str]) -> String {std::iter::once(command.to_string()).chain(args.iter().map(|arg| quote_arg(arg)))diff --git a/tools/dev/cassandra_target_schema.json b/tools/dev/cassandra_target_schema.jsonindex 7aea67521..797962a5e 100644--- a/tools/dev/cassandra_target_schema.json+++ b/tools/dev/cassandra_target_schema.json@@ -9194,92 +9194,6 @@],"primary_key": "((guild_id), webhook_id)","options": ""- },- {- "name": "gateway_hotpatch_events_by_build",- "columns": [- {- "name": "build_sha",- "type": "text"- },- {- "name": "event_id",- "type": "timeuuid"- },- {- "name": "schema_version",- "type": "int"- },- {- "name": "kind",- "type": "text"- },- {- "name": "created_at",- "type": "timestamp"- },- {- "name": "created_by",- "type": "text"- },- {- "name": "signer_key_id",- "type": "text"- },- {- "name": "bundle_sha256",- "type": "blob"- },- {- "name": "signature",- "type": "blob"- },- {- "name": "bundle",- "type": "blob"- }- ],- "primary_key": "((build_sha), event_id)",- "options": "CLUSTERING ORDER BY (event_id ASC)"- },- {- "name": "gateway_hotpatch_applied_by_node",- "columns": [- {- "name": "build_sha",- "type": "text"- },- {- "name": "node_name",- "type": "text"- },- {- "name": "event_id",- "type": "timeuuid"- },- {- "name": "applied_at",- "type": "timestamp"- },- {- "name": "module_count",- "type": "int"- },- {- "name": "bundle_sha256",- "type": "blob"- },- {- "name": "status",- "type": "text"- },- {- "name": "error",- "type": "text"- }- ],- "primary_key": "((build_sha, node_name), event_id)",- "options": "CLUSTERING ORDER BY (event_id ASC)"}],"indexes": [diff --git a/tools/dev/src/bootstrap.rs b/tools/dev/src/bootstrap.rsindex 41879683c..74a4232d9 100644--- a/tools/dev/src/bootstrap.rs+++ b/tools/dev/src/bootstrap.rs@@ -1,18 +1,15 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use crate::desktop::install_desktop;use crate::gateway::setup_gateway_config;-use crate::object_store::{bootstrap_schema_and_object_store, wait_s3_api};+use crate::object_store::bootstrap_schema;use crate::paths::{ensure_state_dirs, ensure_writable_dev_paths};-use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http, wait_tcp};+use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http_success, wait_tcp};use anyhow::Result;-pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result<()> {+pub async fn bootstrap(skip_install: bool) -> Result<()> {ensure_state_dirs()?;ensure_writable_dev_paths()?;if !skip_install {- crate::proc::run(&["corepack", "enable"])?;- crate::proc::run(&["corepack", "prepare", "[email protected]", "--activate"])?;run_command(&["pnpm", "install", "--frozen-lockfile"],RunOptions {@@ -23,13 +20,10 @@ pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result..RunOptions::default()},)?;- if !skip_desktop_install {- install_desktop()?;- }}setup_gateway_config()?;wait_core_infra().await?;- bootstrap_schema_and_object_store().await?;+ bootstrap_schema().await?;println!("Fluxer dev bootstrap complete.");Ok(())}@@ -43,16 +37,11 @@ pub async fn post_start() -> Result<()> {}pub async fn wait_core_infra() -> Result<()> {+ wait_tcp("Postgres", "postgres", 5432, 120).await?;+ wait_http_success("Meilisearch", "http://meilisearch:7700/health", 120).await?;wait_tcp("Valkey", "valkey", 6379, 120).await?;wait_tcp("NATS", "nats", 4222, 120).await?;wait_tcp("LiveKit", "livekit", 7880, 120).await?;crate::media_proxy::ensure_dev_object_store(true, 120).await?;- wait_tcp("SeaweedFS S3", "127.0.0.1", 8333, 120).await?;- wait_http(- "SeaweedFS master",- "http://127.0.0.1:9333/cluster/status",- 120,- )- .await?;- wait_s3_api(120).await+ Ok(())}diff --git a/tools/dev/src/dev.rs b/tools/dev/src/dev.rsindex 6d5769e88..1dc8c832c 100644--- a/tools/dev/src/dev.rs+++ b/tools/dev/src/dev.rs@@ -1,15 +1,16 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use crate::gateway::setup_gateway_config;+use crate::gateway::{build_gateway_cluster_nodes, setup_gateway_config};use crate::manifest::{- ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_PORT, LOOPBACK_HOST,- MEDIA_PROXY_PORT, rust_services,+ ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_GATEWAY_PORTS_ENV,+ DEV_PROXY_PORT, GATEWAY_PORT, LOOPBACK_HOST, MEDIA_PROXY_PORT, rust_services,};use crate::object_store::s3_endpoint;use crate::paths::{DESKTOP_DIR, ROOT};use crate::proc::{- PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command,- merged_env, restart_budget_exceeded, run_command, wait_http,+ AwaitOutcome, PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal,+ await_or_shutdown, configure_process_group, format_command, merged_env, prefix_output,+ restart_budget_exceeded, run_command_interruptible, wait_http,};use anyhow::{Context, Result, bail};use std::collections::{BTreeMap, VecDeque};@@ -30,10 +31,10 @@ const DEFAULT_TASKS: &[&str] = &["proxy","services","media",- "gateway","marketing","admin","api",+ "gateway-single","worker","app","app-proxy",@@ -63,7 +64,7 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i"The private marketing project is unavailable. Authorized maintainers can initialize it with:\n {MARKETING_INITIALIZE_COMMAND}");}- let tasks = task_table_for_availability(marketing_availability)?;+ let mut tasks = task_table_for_availability(marketing_availability)?;let selected = if task_names.is_empty() {DEFAULT_TASKS.iter()@@ -88,11 +89,30 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<iunknown.join(", "));}- ensure_js_dependencies_if_needed(&selected)?;- ensure_object_store_if_needed(&selected).await?;- wait_for_search_backend_if_needed(&selected).await?;- setup_gateway_config()?;+ let gateway_readiness_port = configure_gateway_proxy_ports(&mut tasks, &selected)?;let mut shutdown = ShutdownSignal::new()?;+ match ensure_js_dependencies_if_needed(&selected, &mut shutdown).await? {+ AwaitOutcome::Completed(()) => {}+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev startup...");+ return Ok(0);+ }+ }+ match await_or_shutdown(&mut shutdown, ensure_object_store_if_needed(&selected)).await {+ AwaitOutcome::Completed(result) => result?,+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev startup...");+ return Ok(0);+ }+ }+ match await_or_shutdown(&mut shutdown, wait_for_search_backend_if_needed(&selected)).await {+ AwaitOutcome::Completed(result) => result?,+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev startup...");+ return Ok(0);+ }+ }+ setup_gateway_config()?;let mut processes = Vec::new();let mut task_names = Vec::new();let mut task_specs = Vec::new();@@ -102,29 +122,82 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<ilet mut next_object_store_check = Instant::now() + object_store_monitor_interval;let selected_for_readiness = selected.clone();for name in selected {- if name == CLOUDFLARE_TUNNEL_TASK- && let Err(error) =- wait_for_cloudflare_tunnel_routes(&mut processes, &selected_for_readiness).await- {- crate::gateway::stop_processes(&mut processes);- return Err(error);+ if name == CLOUDFLARE_TUNNEL_TASK {+ match await_or_shutdown(+ &mut shutdown,+ wait_for_cloudflare_tunnel_routes(&mut processes, &selected_for_readiness),+ )+ .await+ {+ AwaitOutcome::Completed(Ok(())) => {}+ AwaitOutcome::Completed(Err(error)) => {+ crate::gateway::stop_processes(&mut processes);+ return Err(error);+ }+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev tasks...");+ crate::gateway::stop_processes(&mut processes);+ return Ok(0);+ }+ }}- let process = start_task(tasks.get(name.as_str()).expect("validated task"))?;+ let process = match start_task(tasks.get(name.as_str()).expect("validated task")) {+ Ok(process) => process,+ Err(error) => {+ crate::gateway::stop_processes(&mut processes);+ return Err(error);+ }+ };processes.push(process);task_names.push(name.clone());task_specs.push(tasks.get(name.as_str()).expect("validated task").clone());task_restarts.push(VecDeque::new());- if name == "services"- && let Err(error) = wait_for_rust_services(&mut processes).await- {- crate::gateway::stop_processes(&mut processes);- return Err(error);+ if name == "services" {+ match await_or_shutdown(&mut shutdown, wait_for_rust_services(&mut processes)).await {+ AwaitOutcome::Completed(Ok(())) => {}+ AwaitOutcome::Completed(Err(error)) => {+ crate::gateway::stop_processes(&mut processes);+ return Err(error);+ }+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev tasks...");+ crate::gateway::stop_processes(&mut processes);+ return Ok(0);+ }+ }}- if name == "api"- && let Err(error) = wait_for_api(&mut processes).await- {- crate::gateway::stop_processes(&mut processes);- return Err(error);+ if name == "gateway" || name == "gateway-single" {+ match await_or_shutdown(+ &mut shutdown,+ wait_for_gateway(&mut processes, gateway_readiness_port),+ )+ .await+ {+ AwaitOutcome::Completed(Ok(())) => {}+ AwaitOutcome::Completed(Err(error)) => {+ crate::gateway::stop_processes(&mut processes);+ return Err(error);+ }+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev tasks...");+ crate::gateway::stop_processes(&mut processes);+ return Ok(0);+ }+ }+ }+ if name == "api" {+ match await_or_shutdown(&mut shutdown, wait_for_api(&mut processes)).await {+ AwaitOutcome::Completed(Ok(())) => {}+ AwaitOutcome::Completed(Err(error)) => {+ crate::gateway::stop_processes(&mut processes);+ return Err(error);+ }+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev tasks...");+ crate::gateway::stop_processes(&mut processes);+ return Ok(0);+ }+ }}}loop {@@ -135,9 +208,17 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<ireturn Err(error);}if monitor_object_store && Instant::now() >= next_object_store_check {- if let Err(error) = monitor_object_store_dependency().await {- crate::gateway::stop_processes(&mut processes);- return Err(error);+ match await_or_shutdown(&mut shutdown, monitor_object_store_dependency()).await {+ AwaitOutcome::Completed(Ok(())) => {}+ AwaitOutcome::Completed(Err(error)) => {+ crate::gateway::stop_processes(&mut processes);+ return Err(error);+ }+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping dev tasks...");+ crate::gateway::stop_processes(&mut processes);+ return Ok(0);+ }}next_object_store_check = Instant::now() + object_store_monitor_interval;}@@ -152,6 +233,38 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i}}+fn configure_gateway_proxy_ports(+ tasks: &mut BTreeMap<&'static str, DevTask>,+ selected: &[String],+) -> Result<u16> {+ let cluster_selected = selected.iter().any(|name| name == "gateway");+ let single_selected = selected.iter().any(|name| name == "gateway-single");+ if cluster_selected && single_selected {+ bail!("Select either `gateway` or `gateway-single`, not both");+ }+ let ports = if cluster_selected {+ build_gateway_cluster_nodes()?+ .into_iter()+ .filter(|node| node.role == "websocket")+ .map(|node| node.http_port)+ .collect::<Vec<_>>()+ } else {+ vec![GATEWAY_PORT]+ };+ let value = ports+ .iter()+ .map(u16::to_string)+ .collect::<Vec<_>>()+ .join(",");+ let proxy = tasks+ .get_mut("proxy")+ .context("dev task table is missing the proxy task")?;+ proxy+ .env+ .push((DEV_PROXY_GATEWAY_PORTS_ENV.to_owned(), Some(value)));+ Ok(*ports.first().expect("gateway proxy ports are non-empty"))+}+async fn ensure_object_store_if_needed(selected: &[String]) -> Result<()> {if !selected_needs_object_store(selected) {return Ok(());@@ -202,6 +315,7 @@ fn restart_exited_tasks(let Some(status) = process.try_wait()? else {continue;};+ crate::gateway::stop_child_processes(&mut [process]);if restart_budget_exceeded(&mut task_restarts[index], Instant::now()) {bail!("Dev task {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",@@ -237,6 +351,10 @@ fn task_table_for_availability(marketing_availability: MarketingAvailability,) -> Result<BTreeMap<&'static str, DevTask>> {let self_tool = self_tool_command()?;+ let api_dir = ROOT.join("fluxer_api");+ let api_tsx = api_dir.join("node_modules/.bin/tsx");+ let app_dir = ROOT.join("fluxer_app");+ let app_dev_server = ROOT.join("tools/ci/run.sh");let public_url = public_url();let marketing_endpoint = std::env::var("FLUXER_MARKETING_ENDPOINT").unwrap_or_else(|_| format!("{public_url}/marketing"));@@ -260,8 +378,13 @@ fn task_table_for_availability(});insert(DevTask {name: "api",- args: strings(&["pnpm", "--filter", "fluxer_api", "dev"]),- cwd: ROOT.clone(),+ args: vec![+ api_tsx.display().to_string(),+ "watch".to_owned(),+ "--clear-screen=false".to_owned(),+ "src/AppEntrypoint.ts".to_owned(),+ ],+ cwd: api_dir.clone(),env: vec![("FLUXER_S3_PUBLIC_ENDPOINT".to_owned(),@@ -286,29 +409,26 @@ fn task_table_for_availability(});insert(DevTask {name: "worker",- args: strings(&[- "pnpm",- "--filter",- "fluxer_api",- "exec",- "tsx",- "watch",- "--clear-screen=false",- "src/WorkerEntrypoint.ts",- ]),- cwd: ROOT.clone(),+ args: vec![+ api_tsx.display().to_string(),+ "watch".to_owned(),+ "--clear-screen=false".to_owned(),+ "src/WorkerEntrypoint.ts".to_owned(),+ ],+ cwd: api_dir,env: Vec::new(),});insert(DevTask {name: "app",- args: strings(&["pnpm", "--filter", "fluxer_app", "dev"]),- cwd: ROOT.clone(),+ args: vec![+ app_dev_server.display().to_string(),+ "app-dev-server".to_owned(),+ ],+ cwd: app_dir,env: vec![+ ("TOKIO_WORKER_THREADS".to_owned(), Some("4".to_owned())),+ ("RAYON_NUM_THREADS".to_owned(), Some("4".to_owned())),("FLUXER_APP_DEV_PORT".to_owned(), Some(APP_PORT.to_string())),- (- "FLUXER_APP_SKIP_I18N_COMPILE".to_owned(),- Some("true".to_owned()),- ),("FLUXER_STATIC_CDN_ENDPOINT".to_owned(),Some(public_url.clone()),@@ -584,27 +704,39 @@ async fn wait_for_search_backend_if_needed(selected: &[String]) -> Result<()> {return Ok(());}let env = merged_env(None, true)?;+ let backend = SearchBackend::from_env(&env)?;let search_url = env.get("FLUXER_SEARCH_URL")- .cloned()- .unwrap_or_else(|| default_search_url(&env));- wait_http(search_backend_label(&env), &search_url, 120).await+ .filter(|url| !url.trim().is_empty())+ .context("Missing FLUXER_SEARCH_URL for the selected search backend")?;+ wait_http(backend.label(), search_url, 120).await}-fn default_search_url(env: &BTreeMap<String, String>) -> String {- if env- .get("FLUXER_SEARCH_ENGINE")- .is_some_and(|engine| engine == "meilisearch")- {- return "http://127.0.0.1:7700".to_owned();- }- "http://127.0.0.1:9200".to_owned()+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum SearchBackend {+ Elasticsearch,+ Meilisearch,}-fn search_backend_label(env: &BTreeMap<String, String>) -> &'static str {- match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) {- Some("meilisearch") => "Meilisearch",- _ => "Elasticsearch",+impl SearchBackend {+ fn from_env(env: &BTreeMap<String, String>) -> Result<Self> {+ match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) {+ Some("elasticsearch") => Ok(Self::Elasticsearch),+ Some("meilisearch") => Ok(Self::Meilisearch),+ Some(engine) => bail!(+ "Unsupported FLUXER_SEARCH_ENGINE value {engine:?}; expected `meilisearch` or `elasticsearch`"+ ),+ None => {+ bail!("Missing FLUXER_SEARCH_ENGINE; expected `meilisearch` or `elasticsearch`")+ }+ }+ }++ fn label(self) -> &'static str {+ match self {+ Self::Elasticsearch => "Elasticsearch",+ Self::Meilisearch => "Meilisearch",+ }}}@@ -627,9 +759,12 @@ fn public_url() -> String {.unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}"))}-fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {+async fn ensure_js_dependencies_if_needed(+ selected: &[String],+ shutdown: &mut ShutdownSignal,+) -> Result<AwaitOutcome<()>> {if selected_needs_js_dependency_preflight(selected) {- run_command(+ match run_command_interruptible(&["pnpm", "install", "--frozen-lockfile"],RunOptions {env: PNPM_INSTALL_ENV@@ -638,11 +773,17 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {.collect(),..RunOptions::default()},- )?;+ shutdown,+ )+ .await?+ {+ AwaitOutcome::Completed(_) => {}+ AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)),+ }}if selected.iter().any(|name| name == "marketing") {let marketing_dir = ROOT.join("fluxer_marketing");- run_command(+ match run_command_interruptible(&["pnpm", "install", "--frozen-lockfile"],RunOptions {cwd: &marketing_dir,@@ -652,9 +793,15 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {.collect(),..RunOptions::default()},- )?;+ shutdown,+ )+ .await?+ {+ AwaitOutcome::Completed(_) => {}+ AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)),+ }}- Ok(())+ Ok(AwaitOutcome::Completed(()))}fn selected_needs_js_dependency_preflight(selected: &[String]) -> bool {@@ -689,14 +836,7 @@ fn start_task(task: &DevTask) -> Result<Child> {.envs(env).stdout(Stdio::piped()).stderr(Stdio::piped());- #[cfg(unix)]- unsafe {- use std::os::unix::process::CommandExt;- command.pre_exec(|| {- libc::setsid();- Ok(())- });- }+ configure_process_group(&mut command);let mut child = command.spawn()?;if let Some(stdout) = child.stdout.take() {let label = task.name.to_owned();@@ -718,13 +858,16 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> {.len().checked_sub(1).expect("services process was just pushed");+ let deadline = Instant::now() + Duration::from_secs(timeout);for spec in rust_services() {for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {- check_startup_processes(processes, services_index, spec.name, mode)?;- wait_http(- &format!("Rust service {}:{mode}", spec.name),+ let name = format!("Rust service {}:{mode}", spec.name);+ wait_http_with_startup_checks_until(+ processes,+ services_index,+ &name,&format!("http://{LOOPBACK_HOST}:{port}/_health"),- timeout,+ deadline,).await?;}@@ -732,6 +875,25 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> {Ok(())}+async fn wait_for_gateway(processes: &mut [Child], port: u16) -> Result<()> {+ let timeout = std::env::var("FLUXER_DEV_GATEWAY_READY_TIMEOUT")+ .ok()+ .and_then(|value| value.parse().ok())+ .unwrap_or(180);+ let gateway_index = processes+ .len()+ .checked_sub(1)+ .expect("gateway process was just pushed");+ wait_http_with_startup_checks(+ processes,+ gateway_index,+ "Gateway",+ &format!("http://{LOOPBACK_HOST}:{port}/_health/ready"),+ timeout,+ )+ .await+}+async fn wait_for_api(processes: &mut [Child]) -> Result<()> {let timeout = std::env::var("FLUXER_DEV_API_READY_TIMEOUT").ok()@@ -827,7 +989,7 @@ async fn wait_http_for_dev_tasks(while Instant::now() < deadline {check_dev_startup_processes(processes, name)?;match client.get(url).send().await {- Ok(response) if response.status().as_u16() < 500 => {+ Ok(response) if response.status().is_success() => {println!("{name} is reachable at {url}");return Ok(());}@@ -849,16 +1011,32 @@ async fn wait_http_with_startup_checks(name: &str,url: &str,timeout_secs: u64,+) -> Result<()> {+ wait_http_with_startup_checks_until(+ processes,+ watched_index,+ name,+ url,+ Instant::now() + Duration::from_secs(timeout_secs),+ )+ .await+}++async fn wait_http_with_startup_checks_until(+ processes: &mut [Child],+ watched_index: usize,+ name: &str,+ url: &str,+ deadline: Instant,) -> Result<()> {let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build()?;- let deadline = Instant::now() + Duration::from_secs(timeout_secs);let mut last_error = None;while Instant::now() < deadline {check_startup_task_processes(processes, watched_index, name)?;match client.get(url).send().await {- Ok(response) if response.status().as_u16() < 500 => {+ Ok(response) if response.status().is_success() => {println!("{name} is reachable at {url}");return Ok(());}@@ -901,35 +1079,6 @@ fn check_startup_task_processes(Ok(())}-fn check_startup_processes(- processes: &mut [Child],- services_index: usize,- service_name: &str,- mode: &str,-) -> Result<()> {- for (index, process) in processes.iter_mut().enumerate() {- if let Some(status) = process.try_wait()? {- let code = status.code().unwrap_or(1);- if index == services_index {- bail!(- "Rust service supervisor exited with status {code} before {service_name}:{mode} became ready"- );- }- bail!(- "Dev task exited with status {code} before Rust service {service_name}:{mode} became ready"- );- }- }- Ok(())-}--fn prefix_output(label: &str, reader: impl std::io::Read) {- use std::io::{BufRead, BufReader};- for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {- println!("[{label}] {line}");- }-}-#[allow(dead_code)]fn _cwd_is_root(path: &Path) -> bool {path == ROOT.as_path()@@ -940,17 +1089,17 @@ mod tests {use super::*;#[test]- fn default_tasks_keep_legacy_order() {+ fn default_tasks_use_lightweight_gateway() {assert_eq!(DEFAULT_TASKS,&["proxy","services","media",- "gateway","marketing","admin","api",+ "gateway-single","worker","app","app-proxy"diff --git a/tools/dev/src/gateway.rs b/tools/dev/src/gateway.rsindex a06644bb7..db8cf2956 100644--- a/tools/dev/src/gateway.rs+++ b/tools/dev/src/gateway.rs@@ -1,18 +1,20 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use crate::gateway_reload::{- ArtifactState, changed_artifacts, hot_reload_enabled, hot_reload_modules, snapshot_artifacts,- spawn_source_watcher,-};use crate::paths::{DEV_GATEWAY_DIR, GATEWAY_CONFIG_DIR, ROOT};use crate::proc::{- RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,- restart_budget_exceeded, run_command,+ AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown,+ configure_process_group, force_kill_process_group, format_command, merged_env,+ pending_shutdown, prefix_output, process_group_running, restart_budget_exceeded,+ run_command_interruptible, stop_process_group, terminate_process_group,};use anyhow::{Context, Result, bail};+use sha2::{Digest, Sha256};+#[cfg(target_os = "linux")]+use std::collections::BTreeSet;use std::collections::VecDeque;use std::env;use std::fs;+use std::io::Read;use std::net::{SocketAddr, TcpListener};use std::path::{Path, PathBuf};use std::process::{Child, Command, Stdio};@@ -47,9 +49,14 @@ const GATEWAY_COMPILE_COMMAND: &[&str] = &["--step","compile",];+const GATEWAY_DEPENDENCY_INPUTS: &[&str] = &["rebar.config", "rebar.config.script", "rebar.lock"];+const GATEWAY_DEPENDENCY_INPUT_MAX_BYTES: u64 = 4 * 1024 * 1024;+const GATEWAY_DEPENDENCY_STAMP_FILE: &str = "rebar-dependencies.sha256";+const GATEWAY_DEPENDENCY_STAMP_MAX_BYTES: u64 = 128;const NODE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);const NODE_RESTART_PORT_WAIT: Duration = Duration::from_secs(75);-const STOP_GRACE_PERIOD: Duration = Duration::from_secs(10);+const NESTED_STOP_GRACE_PERIOD: Duration = Duration::from_secs(5);+const ROOT_STOP_GRACE_PERIOD: Duration = Duration::from_secs(15);#[derive(Debug, Clone, PartialEq, Eq)]pub struct GatewayNode {@@ -128,33 +135,42 @@ fn remove_stale_gateway_config() -> Result<()> {Ok(())}-pub fn run_gateway() -> Result<()> {+pub async fn run_gateway() -> Result<i32> {+ let mut shutdown = ShutdownSignal::new()?;+ let node_names = std::collections::HashSet::from([env::var("FLUXER_ERLANG_NODE_NAME")+ .unwrap_or_else(|_| "[email protected]".to_owned())]);+ cleanup_orphaned_gateway_processes(&node_names).await?;+ if let Some(signal) = pending_shutdown(&mut shutdown).await {+ println!("Received {signal}; stopping gateway startup...");+ return Ok(0);+ }setup_gateway_config()?;- compile_gateway()?;- let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?;- let env = merged_env(None, true)?;- println!("$ {}", format_command(&command));-- #[cfg(unix)]- {- use std::os::unix::process::CommandExt;- let err = Command::new(&command[0])- .args(&command[1..])- .current_dir(gateway_dir())- .env_clear()- .envs(env)- .exec();- bail!("failed to exec gateway: {err}");+ match compile_gateway_interruptible(&mut shutdown).await? {+ AwaitOutcome::Completed(_) => {}+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping gateway startup...");+ return Ok(0);+ }}- #[cfg(not(unix))]- {- let status = Command::new(&command[0])- .args(&command[1..])- .current_dir(gateway_dir())- .env_clear()- .envs(env)- .status()?;- std::process::exit(status.code().unwrap_or(1));+ let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?;+ let command_refs = command.iter().map(String::as_str).collect::<Vec<_>>();+ let working_dir = gateway_dir();+ let outcome = run_command_interruptible(+ &command_refs,+ RunOptions {+ cwd: &working_dir,+ ..RunOptions::default()+ },+ &mut shutdown,+ )+ .await?;+ stop_idle_epmd();+ match outcome {+ AwaitOutcome::Completed(output) => Ok(output.status.code().unwrap_or(1)),+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping gateway...");+ Ok(0)+ }}}@@ -166,11 +182,31 @@ struct SupervisedNode {pub async fn run_gateway_cluster() -> Result<i32> {let nodes = build_gateway_cluster_nodes()?;- cleanup_orphaned_gateway_nodes(&nodes).await?;- wait_for_cluster_ports_available(&nodes).await?;- setup_gateway_cluster_config(&nodes)?;- compile_gateway()?;let mut shutdown = ShutdownSignal::new()?;+ let node_names = nodes+ .iter()+ .map(GatewayNode::erlang_name)+ .collect::<std::collections::HashSet<_>>();+ cleanup_orphaned_gateway_processes(&node_names).await?;+ if let Some(signal) = pending_shutdown(&mut shutdown).await {+ println!("Received {signal}; stopping gateway startup...");+ return Ok(0);+ }+ match await_or_shutdown(&mut shutdown, wait_for_cluster_ports_available(&nodes)).await {+ AwaitOutcome::Completed(result) => result?,+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping gateway startup...");+ return Ok(0);+ }+ }+ setup_gateway_cluster_config(&nodes)?;+ match compile_gateway_interruptible(&mut shutdown).await? {+ AwaitOutcome::Completed(_) => {}+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping gateway startup...");+ return Ok(0);+ }+ }let static_peers = nodes.iter().map(GatewayNode::erlang_name)@@ -179,57 +215,35 @@ pub async fn run_gateway_cluster() -> Result<i32> {let mut supervised = Vec::new();print_gateway_cluster_topology(&nodes);for node in &nodes {+ let child = match start_node(node, &static_peers) {+ Ok(child) => child,+ Err(error) => {+ stop_supervised(&mut supervised);+ return Err(error);+ }+ };supervised.push(SupervisedNode {node: node.clone(),- child: start_node(node, &static_peers)?,+ child,restarts: VecDeque::new(),});}- let watcher = hot_reload_enabled().then(spawn_source_watcher);- if watcher.is_some() {- println!(- "Gateway hot reload enabled; watching fluxer_gateway sources (set FLUXER_DEV_GATEWAY_HOT_RELOAD=false to disable)"- );- }- let mut compile: Option<(tokio::task::JoinHandle<bool>, ArtifactState)> = None;- let mut compile_queued = false;loop {- if let Err(error) = restart_exited_nodes(&mut supervised, &static_peers).await {- stop_supervised(&mut supervised);- return Err(error);- }- if let Some(receiver) = &watcher {- while receiver.try_recv().is_ok() {- compile_queued = true;- }- }- if compile_queued && compile.is_none() {- compile_queued = false;- println!("Gateway sources changed; recompiling for hot reload...");- let before = snapshot_artifacts();- compile = Some((- tokio::task::spawn_blocking(compile_gateway_for_reload),- before,- ));- }- if compile- .as_ref()- .is_some_and(|(handle, _)| handle.is_finished())+ match await_or_shutdown(+ &mut shutdown,+ restart_exited_nodes(&mut supervised, &static_peers),+ )+ .await{- let (handle, before) = compile.take().expect("compile task present");- match handle.await {- Ok(true) => {- if let Err(error) =- apply_hot_reload(&mut supervised, &before, &static_peers).await- {- stop_supervised(&mut supervised);- return Err(error);- }- }- Ok(false) => println!(- "Gateway compile failed; hot reload skipped (fix the errors and save again)"- ),- Err(error) => println!("Gateway compile task failed: {error}"),+ AwaitOutcome::Completed(Ok(())) => {}+ AwaitOutcome::Completed(Err(error)) => {+ stop_supervised(&mut supervised);+ return Err(error);+ }+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping gateway cluster...");+ stop_supervised(&mut supervised);+ return Ok(0);}}tokio::select! {@@ -243,24 +257,116 @@ pub async fn run_gateway_cluster() -> Result<i32> {}}-fn compile_gateway() -> Result<()> {- run_command(GATEWAY_COMPILE_COMMAND, RunOptions::default()).map(drop)+async fn compile_gateway_interruptible(+ shutdown: &mut ShutdownSignal,+) -> Result<AwaitOutcome<std::process::Output>> {+ let dependency_stamp = prepare_gateway_compile()?;+ let outcome =+ run_command_interruptible(GATEWAY_COMPILE_COMMAND, RunOptions::default(), shutdown).await?;+ if matches!(&outcome, AwaitOutcome::Completed(_)) {+ write_gateway_dependency_stamp(&dependency_stamp)?;+ }+ Ok(outcome)}-fn compile_gateway_for_reload() -> bool {- run_command(- GATEWAY_COMPILE_COMMAND,- RunOptions {- check: false,- ..RunOptions::default()- },- )- .map(|output| output.status.success())- .unwrap_or(false)+fn prepare_gateway_compile() -> Result<String> {+ let dependency_stamp = gateway_dependency_stamp()?;+ if gateway_dependency_stamp_matches(&dependency_stamp)? {+ return Ok(dependency_stamp);+ }+ let build_dir = gateway_dir().join("_build/default");+ println!(+ "Gateway dependency inputs changed; removing stale {}",+ build_dir.display()+ );+ match fs::remove_dir_all(&build_dir) {+ Ok(()) => {}+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}+ Err(error) => {+ return Err(error).with_context(|| format!("failed to remove {}", build_dir.display()));+ }+ }+ Ok(dependency_stamp)}-fn cluster_cookie() -> String {- env::var("FLUXER_ERLANG_COOKIE").unwrap_or_else(|_| GATEWAY_CLUSTER_COOKIE.to_owned())+fn gateway_dependency_stamp() -> Result<String> {+ let mut hasher = Sha256::new();+ let mut total_bytes = 0_u64;+ for input in GATEWAY_DEPENDENCY_INPUTS {+ let path = gateway_dir().join(input);+ let metadata = fs::metadata(&path)+ .with_context(|| format!("failed to read metadata for {}", path.display()))?;+ if !metadata.is_file() {+ bail!("gateway dependency input is not a file: {}", path.display());+ }+ total_bytes = total_bytes+ .checked_add(metadata.len())+ .context("gateway dependency input byte count overflowed")?;+ if total_bytes > GATEWAY_DEPENDENCY_INPUT_MAX_BYTES {+ bail!(+ "gateway dependency inputs exceed {} bytes",+ GATEWAY_DEPENDENCY_INPUT_MAX_BYTES+ );+ }+ let mut bytes = Vec::with_capacity(metadata.len() as usize);+ fs::File::open(&path)+ .with_context(|| format!("failed to open {}", path.display()))?+ .take(metadata.len() + 1)+ .read_to_end(&mut bytes)+ .with_context(|| format!("failed to read {}", path.display()))?;+ if bytes.len() as u64 != metadata.len() {+ bail!(+ "gateway dependency input changed while reading: {}",+ path.display()+ );+ }+ hasher.update((input.len() as u64).to_le_bytes());+ hasher.update(input.as_bytes());+ hasher.update(metadata.len().to_le_bytes());+ hasher.update(&bytes);+ }+ Ok(bytes_to_lower_hex(&hasher.finalize()))+}++fn bytes_to_lower_hex(bytes: &[u8]) -> String {+ const HEX: &[u8; 16] = b"0123456789abcdef";+ let mut output = String::with_capacity(bytes.len() * 2);+ for byte in bytes {+ output.push(HEX[(byte >> 4) as usize] as char);+ output.push(HEX[(byte & 0x0f) as usize] as char);+ }+ output+}++fn gateway_dependency_stamp_matches(expected: &str) -> Result<bool> {+ let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE);+ let file = match fs::File::open(&path) {+ Ok(file) => file,+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),+ Err(error) => {+ return Err(error).with_context(|| format!("failed to open {}", path.display()));+ }+ };+ let mut bytes = Vec::new();+ file.take(GATEWAY_DEPENDENCY_STAMP_MAX_BYTES + 1)+ .read_to_end(&mut bytes)+ .with_context(|| format!("failed to read {}", path.display()))?;+ if bytes.len() as u64 > GATEWAY_DEPENDENCY_STAMP_MAX_BYTES {+ bail!(+ "gateway dependency stamp exceeds {} bytes: {}",+ GATEWAY_DEPENDENCY_STAMP_MAX_BYTES,+ path.display()+ );+ }+ Ok(bytes == format!("{expected}\n").as_bytes())+}++fn write_gateway_dependency_stamp(dependency_stamp: &str) -> Result<()> {+ fs::create_dir_all(DEV_GATEWAY_DIR.as_path())+ .with_context(|| format!("failed to create {}", DEV_GATEWAY_DIR.display()))?;+ let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE);+ fs::write(&path, format!("{dependency_stamp}\n"))+ .with_context(|| format!("failed to write {}", path.display()))}async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: &str) -> Result<()> {@@ -285,18 +391,7 @@ async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: &}async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result<()> {- if entry.child.try_wait()?.is_none() {- terminate_process(&mut entry.child);- let deadline = Instant::now() + NODE_SHUTDOWN_TIMEOUT;- while entry.child.try_wait()?.is_none() {- if Instant::now() >= deadline {- let _ = entry.child.kill();- let _ = entry.child.wait();- break;- }- sleep(Duration::from_millis(100)).await;- }- }+ stop_process_group(&mut entry.child, NODE_SHUTDOWN_TIMEOUT).await;wait_for_ports_available_until(std::slice::from_ref(&entry.node),Instant::now() + NODE_RESTART_PORT_WAIT,@@ -306,79 +401,13 @@ async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result<Ok(())}-async fn apply_hot_reload(- supervised: &mut [SupervisedNode],- before: &ArtifactState,- static_peers: &str,-) -> Result<()> {- let after = snapshot_artifacts();- let diff = changed_artifacts(before, &after);- if diff.nifs_changed {- println!("Gateway native NIF artifacts changed; rolling restart of all gateway nodes...");- for entry in supervised.iter_mut() {- println!("[gateway:{}] restarting for NIF reload", entry.node.name());- restart_node(entry, static_peers).await?;- }- println!("Gateway rolling restart complete");- return Ok(());- }- if diff.modules.is_empty() {- println!("Gateway compile finished; no module changes to reload");- return Ok(());- }- println!(- "Hot reloading {} gateway module(s) across {} node(s): {}",- diff.modules.len(),- supervised.len(),- diff.modules.join(", ")- );- let nodes = supervised- .iter()- .map(|entry| entry.node.clone())- .collect::<Vec<_>>();- let modules = diff.modules.clone();- let cookie = cluster_cookie();- let outcome =- tokio::task::spawn_blocking(move || hot_reload_modules(&nodes, &modules, &cookie)).await;- match outcome {- Ok(Ok(outcome)) if outcome.failed_nodes.is_empty() => {- println!(- "Gateway hot reload complete ({} module(s) live)",- diff.modules.len()- );- }- Ok(Ok(outcome)) => {- for entry in supervised.iter_mut() {- if !outcome.failed_nodes.contains(&entry.node.erlang_name()) {- continue;- }- if entry.child.try_wait()?.is_some() {- continue;- }- println!(- "[gateway:{}] hot reload failed; restarting node to pick up new code",- entry.node.name()- );- restart_node(entry, static_peers).await?;- }- }- Ok(Err(error)) => {- println!("Gateway hot reload failed: {error}; rolling restart of all gateway nodes");- for entry in supervised.iter_mut() {- restart_node(entry, static_peers).await?;- }- }- Err(error) => println!("Gateway hot reload task failed: {error}"),- }- Ok(())-}-fn stop_supervised(supervised: &mut [SupervisedNode]) {let mut children = supervised.iter_mut().map(|entry| &mut entry.child).collect::<Vec<_>>();stop_child_processes(&mut children);+ stop_idle_epmd();}pub fn build_gateway_cluster_nodes() -> Result<Vec<GatewayNode>> {@@ -492,18 +521,20 @@ fn can_bind_tcp_port(port: u16) -> bool {}#[cfg(target_os = "linux")]-async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> {- let leaders = orphaned_gateway_leaders(nodes)?;+async fn cleanup_orphaned_gateway_processes(+ node_names: &std::collections::HashSet<String>,+) -> Result<()> {+ let leaders = orphaned_gateway_leaders(node_names)?;if leaders.is_empty() {return Ok(());}let pids = leaders.iter().map(|leader| leader.pid).collect::<Vec<_>>();println!(- "Stopping orphaned gateway cluster node process group(s): {}",+ "Stopping orphaned gateway process group(s): {}",format_pids(&pids));- let term_failed = signal_process_groups(&pids, libc::SIGTERM);+ let term_failed = signal_process_groups(&leaders, libc::SIGTERM);if !term_failed.is_empty() {println!("SIGTERM delivery failed for orphaned gateway pid(s): {}",@@ -518,7 +549,12 @@ async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> {return Ok(());}if Instant::now() >= deadline {- let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);+ let remaining_leaders = leaders+ .iter()+ .filter(|leader| remaining.contains(&leader.pid))+ .cloned()+ .collect::<Vec<_>>();+ let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL);sleep(Duration::from_millis(200)).await;let survivors = surviving_gateway_group_pids(&leaders)?;if survivors.is_empty() {@@ -556,24 +592,95 @@ fn format_pids(pids: &[i32]) -> String {}#[cfg(not(target_os = "linux"))]-async fn cleanup_orphaned_gateway_nodes(_nodes: &[GatewayNode]) -> Result<()> {+async fn cleanup_orphaned_gateway_processes(+ _node_names: &std::collections::HashSet<String>,+) -> Result<()> {Ok(())}#[cfg(target_os = "linux")]-#[derive(Clone, Copy, Debug, Eq, PartialEq)]+#[derive(Clone, Debug, Eq, PartialEq)]struct GatewayLeader {+ pid: i32,+ members: Vec<GatewayProcessIdentity>,+}++#[cfg(target_os = "linux")]+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+struct GatewayProcessIdentity {pid: i32,starttime: u64,}#[cfg(target_os = "linux")]-fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result<Vec<GatewayLeader>> {- let node_names = nodes+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+struct GatewayProcess {+ pid: i32,+ ppid: i32,+ pgid: i32,+ state: char,+ starttime: u64,+}++#[cfg(target_os = "linux")]+fn orphaned_gateway_leaders(+ node_names: &std::collections::HashSet<String>,+) -> Result<Vec<GatewayLeader>> {+ let current_exe = std::env::current_exe().context("failed to resolve fluxer-dev executable")?;+ let processes = gateway_process_snapshot()?;+ let mut owned_pids = BTreeSet::new();+ for process in &processes {+ if process.ppid != 1 || proc_stat_state_is_dead(process.state) {+ continue;+ }+ let args = proc_cmdline(process.pid);+ let orphaned_supervisor = cmdline_is_gateway_supervisor(&args, ¤t_exe);+ let orphaned_node = (cmdline_has_gateway_node(&args, node_names)+ || cmdline_has_managed_gateway_config(&args))+ && fs::read_link(format!("/proc/{}/cwd", process.pid))+ .ok()+ .as_deref()+ == Some(gateway_dir().as_path());+ if orphaned_supervisor || orphaned_node {+ owned_pids.insert(process.pid);+ }+ }+ loop {+ let previous_count = owned_pids.len();+ for process in &processes {+ if owned_pids.contains(&process.ppid) {+ owned_pids.insert(process.pid);+ }+ }+ if owned_pids.len() == previous_count {+ break;+ }+ }+ let group_pids = processes.iter()- .map(GatewayNode::erlang_name)- .collect::<std::collections::HashSet<_>>();- let mut leaders = Vec::new();+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1)+ .map(|process| process.pgid)+ .collect::<BTreeSet<_>>();+ let leaders = group_pids+ .into_iter()+ .map(|pid| GatewayLeader {+ pid,+ members: processes+ .iter()+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid)+ .map(|process| GatewayProcessIdentity {+ pid: process.pid,+ starttime: process.starttime,+ })+ .collect(),+ })+ .collect::<Vec<_>>();+ Ok(leaders)+}++#[cfg(target_os = "linux")]+fn gateway_process_snapshot() -> Result<Vec<GatewayProcess>> {+ let mut processes = Vec::new();for entry in fs::read_dir("/proc").context("failed to read /proc")? {let Ok(entry) = entry else {continue;@@ -585,18 +692,39 @@ fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result<Vec<GatewayLeader>>else {continue;};- if !cmdline_has_gateway_node(&proc_cmdline(pid), &node_names) {+ let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {continue;- }- if proc_parent_pid(pid) != Some(1) {+ };+ let Some((state, ppid, pgid, starttime)) = parse_proc_stat_process(&stat) else {continue;- }- if let Some(starttime) = proc_stat_starttime(pid) {- leaders.push(GatewayLeader { pid, starttime });- }+ };+ processes.push(GatewayProcess {+ pid,+ ppid,+ pgid,+ state,+ starttime,+ });}- leaders.sort_unstable_by_key(|leader| leader.pid);- Ok(leaders)+ Ok(processes)+}++#[cfg(target_os = "linux")]+fn cmdline_is_gateway_supervisor(args: &[String], current_exe: &Path) -> bool {+ args.first().map(Path::new) == Some(current_exe)+ && args.get(1).map(String::as_str) == Some("gateway")+ && match args.get(2).map(String::as_str) {+ None | Some("cluster" | "single") => args.len() <= 3,+ Some(_) => false,+ }+}++#[cfg(target_os = "linux")]+fn cmdline_has_managed_gateway_config(args: &[String]) -> bool {+ args.windows(2).any(|pair| {+ matches!(pair[0].as_str(), "-config" | "-args_file")+ && Path::new(&pair[1]).starts_with(DEV_GATEWAY_DIR.as_path())+ })}#[cfg(any(target_os = "linux", test))]@@ -618,30 +746,6 @@ fn proc_cmdline(pid: i32) -> Vec<String> {.collect()}-#[cfg(target_os = "linux")]-fn proc_parent_pid(pid: i32) -> Option<i32> {- fs::read_to_string(format!("/proc/{pid}/status"))- .ok()?- .lines()- .find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())-}--#[cfg(target_os = "linux")]-fn process_exists(pid: i32) -> bool {- assert!(pid > 0);- match proc_stat_state_and_pgid(pid) {- Some((state, _pgid)) => !proc_stat_state_is_dead(state),- None => false,- }-}--#[cfg(target_os = "linux")]-fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {- assert!(pid > 0);- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;- parse_proc_stat_state_and_pgid(&stat)-}-#[cfg(any(target_os = "linux", test))]fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {let (_, after_comm) = stat.rsplit_once(')')?;@@ -653,10 +757,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {}#[cfg(target_os = "linux")]-fn proc_stat_starttime(pid: i32) -> Option<u64> {- assert!(pid > 0);- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;- parse_proc_stat_starttime(&stat)+fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> {+ let (_, after_comm) = stat.rsplit_once(')')?;+ let mut fields = after_comm.split_ascii_whitespace();+ let state = fields.next()?.chars().next()?;+ let ppid = fields.next()?.parse().ok()?;+ let pgid = fields.next()?.parse().ok()?;+ Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?))}#[cfg(any(target_os = "linux", test))]@@ -672,62 +779,43 @@ fn proc_stat_state_is_dead(state: char) -> bool {#[cfg(target_os = "linux")]fn surviving_gateway_group_pids(leaders: &[GatewayLeader]) -> Result<Vec<i32>> {- let leader_pids = leaders+ Ok(leaders.iter()+ .filter(|leader| gateway_group_has_owned_member(leader)).map(|leader| leader.pid)- .collect::<std::collections::HashSet<_>>();- let mut survivors = Vec::new();- for entry in fs::read_dir("/proc").context("failed to read /proc")? {- let Ok(entry) = entry else {- continue;- };- let Some(pid) = entry- .file_name()- .to_str()- .and_then(|name| name.parse::<i32>().ok())- else {- continue;- };- let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {- continue;+ .collect())+}++#[cfg(target_os = "linux")]+fn gateway_group_has_owned_member(leader: &GatewayLeader) -> bool {+ leader.members.iter().any(|member| {+ let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else {+ return false;};let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {- continue;+ return false;};- if proc_stat_state_is_dead(state) {- continue;- }- if leader_pids.contains(&pgid) {- survivors.push(pid);- continue;- }- let leader = leaders.iter().find(|leader| leader.pid == pid);- if let Some(leader) = leader- && parse_proc_stat_starttime(&stat) == Some(leader.starttime)- {- survivors.push(pid);- }- }- survivors.sort_unstable();- Ok(survivors)+ !proc_stat_state_is_dead(state)+ && pgid == leader.pid+ && parse_proc_stat_starttime(&stat) == Some(member.starttime)+ })}#[cfg(target_os = "linux")]-fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {+fn signal_process_groups(leaders: &[GatewayLeader], signal: i32) -> Vec<i32> {assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);- let mut failed = Vec::with_capacity(pids.len());- for pid in pids {- assert!(*pid > 0);- let group_result = unsafe { libc::kill(-pid, signal) };- if group_result == 0 {+ let mut failed = Vec::with_capacity(leaders.len());+ for leader in leaders {+ assert!(leader.pid > 1);+ if !gateway_group_has_owned_member(leader) {continue;}- let direct_result = unsafe { libc::kill(*pid, signal) };- if direct_result == 0 {+ let group_result = unsafe { libc::kill(-leader.pid, signal) };+ if group_result == 0 {continue;}- if process_exists(*pid) {- failed.push(*pid);+ if gateway_group_has_owned_member(leader) {+ failed.push(leader.pid);}}failed@@ -812,14 +900,7 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result<Child> {.envs(env).stdout(Stdio::piped()).stderr(Stdio::piped());- #[cfg(unix)]- unsafe {- use std::os::unix::process::CommandExt;- child_command.pre_exec(|| {- libc::setsid();- Ok(())- });- }+ configure_process_group(&mut child_command);let mut child = child_command.spawn()?;if let Some(stdout) = child.stdout.take() {let label = node.name();@@ -832,30 +913,30 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result<Child> {Ok(child)}-fn prefix_output(label: &str, reader: impl std::io::Read) {- use std::io::{BufRead, BufReader};- for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {- println!("[{label}] {line}");- }-}-pub fn stop_processes(processes: &mut [Child]) {let mut children = processes.iter_mut().collect::<Vec<_>>();- stop_child_processes(&mut children);+ stop_child_processes_with_grace(&mut children, ROOT_STOP_GRACE_PERIOD);+ stop_idle_epmd();}pub fn stop_child_processes(processes: &mut [&mut Child]) {+ stop_child_processes_with_grace(processes, NESTED_STOP_GRACE_PERIOD);+}++fn stop_child_processes_with_grace(processes: &mut [&mut Child], grace_period: Duration) {for process in processes.iter_mut() {- if process.try_wait().ok().flatten().is_some() {- continue;+ if let Err(error) = terminate_process_group(process) {+ eprintln!(+ "Failed to terminate process group {}: {error}",+ process.id()+ );}- terminate_process(process);}- let deadline = Instant::now() + STOP_GRACE_PERIOD;+ let deadline = Instant::now() + grace_period;loop {- let all_exited = processes- .iter_mut()- .all(|process| process.try_wait().ok().flatten().is_some());+ let all_exited = processes.iter_mut().all(|process| {+ process.try_wait().ok().flatten().is_some() && !process_group_running(process)+ });if all_exited {return;}@@ -865,38 +946,30 @@ pub fn stop_child_processes(processes: &mut [&mut Child]) {std::thread::sleep(Duration::from_millis(100));}for process in processes {- if process.try_wait().ok().flatten().is_some() {- continue;+ if process.try_wait().ok().flatten().is_none() || process_group_running(process) {+ force_kill_process_group(process);}- force_kill_process(process);}}-#[cfg(unix)]-fn terminate_process(process: &mut Child) {- unsafe {- libc::kill(-(process.id() as i32), libc::SIGTERM);+fn stop_idle_epmd() {+ let output = match Command::new("epmd").arg("-kill").output() {+ Ok(output) => output,+ Err(error) => {+ eprintln!("Failed to stop Erlang port mapper: {error}");+ return;+ }+ };+ if output.status.success() {+ return;}-}--#[cfg(not(unix))]-fn terminate_process(process: &mut Child) {- let _ = process.kill();-}--#[cfg(unix)]-fn force_kill_process(process: &mut Child) {- unsafe {- libc::kill(-(process.id() as i32), libc::SIGKILL);+ let mut message = output.stdout;+ message.extend(output.stderr);+ let message = String::from_utf8_lossy(&message);+ if message.contains("Cannot connect to local epmd") {+ return;}- let _ = process.kill();- let _ = process.wait();-}--#[cfg(not(unix))]-fn force_kill_process(process: &mut Child) {- let _ = process.kill();- let _ = process.wait();+ eprintln!("Failed to stop Erlang port mapper: {}", message.trim());}pub fn build_gateway_command(config_dir: &Path) -> Result<Vec<String>> {diff --git a/tools/dev/src/gateway_reload.rs b/tools/dev/src/gateway_reload.rsdeleted file mode 100644index ee9f694a3..000000000--- a/tools/dev/src/gateway_reload.rs+++ /dev/null@@ -1,420 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--use crate::gateway::{GatewayNode, gateway_dir};-use anyhow::{Context, Result, bail};-use std::collections::{BTreeMap, BTreeSet};-use std::env;-use std::fs;-use std::io::Read;-use std::path::{Path, PathBuf};-use std::process::{Child, Command, Stdio};-use std::sync::mpsc::{Receiver, channel};-use std::time::{Duration, Instant, SystemTime};--const WATCH_POLL_INTERVAL: Duration = Duration::from_millis(1000);-const WATCH_SETTLE_INTERVAL: Duration = Duration::from_millis(300);-const RELOAD_TIMEOUT: Duration = Duration::from_secs(60);-const WATCHED_SOURCE_EXTENSIONS: &[&str] = &["erl", "hrl", "src", "rs", "toml", "config"];--pub type FileState = BTreeMap<PathBuf, (SystemTime, u64)>;--#[derive(Debug, Clone, Default, PartialEq, Eq)]-pub struct ArtifactState {- pub beams: FileState,- pub nifs: FileState,-}--#[derive(Debug, Clone, Default, PartialEq, Eq)]-pub struct ArtifactDiff {- pub modules: Vec<String>,- pub nifs_changed: bool,-}--#[derive(Debug, Clone, Default, PartialEq, Eq)]-pub struct ReloadOutcome {- pub failed_nodes: Vec<String>,-}--pub fn hot_reload_enabled() -> bool {- env::var("FLUXER_DEV_GATEWAY_HOT_RELOAD")- .map(|value| !matches!(value.to_ascii_lowercase().as_str(), "0" | "false" | "no"))- .unwrap_or(true)-}--pub fn spawn_source_watcher() -> Receiver<()> {- let (sender, receiver) = channel();- std::thread::spawn(move || {- let mut previous = scan_sources();- loop {- std::thread::sleep(WATCH_POLL_INTERVAL);- let mut current = scan_sources();- if current == previous {- continue;- }- loop {- std::thread::sleep(WATCH_SETTLE_INTERVAL);- let settled = scan_sources();- if settled == current {- break;- }- current = settled;- }- previous = current;- if sender.send(()).is_err() {- return;- }- }- });- receiver-}--fn scan_sources() -> FileState {- let dir = gateway_dir();- let mut state = FileState::new();- for root in [dir.join("src"), dir.join("include"), dir.join("native")] {- scan_tree(&mut state, &root);- }- for path in [- dir.join("rebar.config"),- dir.join("rebar.config.script"),- dir.join("rebar.lock"),- ] {- record_file(&mut state, &path);- }- state-}--fn scan_tree(state: &mut FileState, root: &Path) {- let Ok(entries) = fs::read_dir(root) else {- return;- };- for entry in entries.flatten() {- let path = entry.path();- let Ok(metadata) = entry.metadata() else {- continue;- };- if metadata.is_dir() {- if !is_skipped_dir(&path) {- scan_tree(state, &path);- }- } else if is_watched_source(&path)- && let Ok(modified) = metadata.modified()- {- state.insert(path, (modified, metadata.len()));- }- }-}--fn record_file(state: &mut FileState, path: &Path) {- if let Ok(metadata) = fs::metadata(path)- && let Ok(modified) = metadata.modified()- {- state.insert(path.to_path_buf(), (modified, metadata.len()));- }-}--fn is_skipped_dir(path: &Path) -> bool {- path.file_name()- .and_then(|name| name.to_str())- .is_some_and(|name| name == "target" || name.starts_with('.'))-}--fn is_watched_source(path: &Path) -> bool {- path.extension()- .and_then(|extension| extension.to_str())- .is_some_and(|extension| WATCHED_SOURCE_EXTENSIONS.contains(&extension))-}--pub fn snapshot_artifacts() -> ArtifactState {- let mut state = ArtifactState::default();- let lib_root = gateway_dir().join("_build/default/lib");- if let Ok(entries) = fs::read_dir(&lib_root) {- for entry in entries.flatten() {- scan_artifact_dir(&mut state.beams, &entry.path().join("ebin"), "beam");- }- }- scan_artifact_dir(&mut state.nifs, &gateway_dir().join("priv"), "so");- state-}--fn scan_artifact_dir(state: &mut FileState, dir: &Path, extension: &str) {- let Ok(entries) = fs::read_dir(dir) else {- return;- };- for entry in entries.flatten() {- let path = entry.path();- if path- .extension()- .is_some_and(|candidate| candidate == extension)- && let Ok(metadata) = entry.metadata()- && let Ok(modified) = metadata.modified()- {- state.insert(path, (modified, metadata.len()));- }- }-}--pub fn changed_artifacts(before: &ArtifactState, after: &ArtifactState) -> ArtifactDiff {- let modules = after- .beams- .iter()- .filter(|(path, state)| before.beams.get(*path) != Some(*state))- .filter_map(|(path, _)| Some(path.file_stem()?.to_str()?.to_owned()))- .collect::<BTreeSet<_>>()- .into_iter()- .collect();- let nifs_changed = after- .nifs- .iter()- .any(|(path, state)| before.nifs.get(path) != Some(state));- ArtifactDiff {- modules,- nifs_changed,- }-}--pub fn build_reload_eval(nodes: &[String], modules: &[String]) -> String {- let node_list = nodes- .iter()- .map(|node| format!("'{node}'"))- .collect::<Vec<_>>()- .join(",");- let module_list = modules- .iter()- .map(|module| format!("'{module}'"))- .collect::<Vec<_>>()- .join(",");- format!(- "Nodes = [{node_list}], \- Mods = [{module_list}], \- Failed = [Node || Node <- Nodes, \- case net_adm:ping(Node) of \- pong -> \- Errors = [Mod || Mod <- Mods, \- begin \- rpc:call(Node, code, purge, [Mod], 10000), \- case rpc:call(Node, code, load_file, [Mod], 10000) of \- {{module, Mod}} -> false; \- Other -> \- io:format(\"gateway_reload_error ~s ~s ~0p~n\", [Node, Mod, Other]), \- true \- end \- end], \- Errors =/= []; \- pang -> \- io:format(\"gateway_reload_node_down ~s~n\", [Node]), \- true \- end], \- [io:format(\"gateway_reload_failed_node ~s~n\", [Failed1]) || Failed1 <- Failed], \- io:format(\"gateway_reload_done~n\"), \- halt(0)."- )-}--pub fn hot_reload_modules(- nodes: &[GatewayNode],- modules: &[String],- cookie: &str,-) -> Result<ReloadOutcome> {- assert!(!modules.is_empty());- let node_names = nodes- .iter()- .map(GatewayNode::erlang_name)- .collect::<Vec<_>>();- let eval = build_reload_eval(&node_names, modules);- let reloader_name = format!("fluxer_dev_reload_{}@127.0.0.1", std::process::id());- let mut child = Command::new("erl")- .args([- "-hidden",- "-noshell",- "-name",- &reloader_name,- "-setcookie",- cookie,- "-eval",- &eval,- ])- .stdin(Stdio::null())- .stdout(Stdio::piped())- .stderr(Stdio::piped())- .spawn()- .context("failed to spawn gateway reload shell")?;- let stdout = child.stdout.take().map(spawn_reader);- let stderr = child.stderr.take().map(spawn_reader);- let status = wait_with_deadline(&mut child, RELOAD_TIMEOUT)?;- let mut output = String::new();- for handle in [stdout, stderr].into_iter().flatten() {- if let Ok(text) = handle.join() {- output.push_str(&text);- }- }- for line in output.lines().filter(|line| !line.trim().is_empty()) {- println!("[gateway:reload] {line}");- }- if !status.success() {- bail!(- "gateway reload shell exited with status {}",- status.code().unwrap_or(1)- );- }- if !output.lines().any(|line| line == "gateway_reload_done") {- bail!("gateway reload shell did not report completion");- }- Ok(parse_reload_outcome(&output))-}--pub fn parse_reload_outcome(output: &str) -> ReloadOutcome {- let failed_nodes = output- .lines()- .filter_map(|line| line.strip_prefix("gateway_reload_failed_node "))- .map(|node| node.trim().to_owned())- .collect::<BTreeSet<_>>()- .into_iter()- .collect();- ReloadOutcome { failed_nodes }-}--fn spawn_reader(stream: impl Read + Send + 'static) -> std::thread::JoinHandle<String> {- std::thread::spawn(move || {- let mut reader = stream;- let mut text = String::new();- let mut bytes = Vec::new();- if reader.read_to_end(&mut bytes).is_ok() {- text = String::from_utf8_lossy(&bytes).into_owned();- }- text- })-}--fn wait_with_deadline(child: &mut Child, timeout: Duration) -> Result<std::process::ExitStatus> {- let deadline = Instant::now() + timeout;- loop {- if let Some(status) = child.try_wait()? {- return Ok(status);- }- if Instant::now() >= deadline {- let _ = child.kill();- let _ = child.wait();- bail!(- "gateway reload shell timed out after {}s",- timeout.as_secs()- );- }- std::thread::sleep(Duration::from_millis(100));- }-}--#[cfg(test)]-mod tests {- use super::*;-- fn file_state(entries: &[(&str, u64, u64)]) -> FileState {- entries- .iter()- .map(|(path, seconds, size)| {- (- PathBuf::from(path),- (- SystemTime::UNIX_EPOCH + Duration::from_secs(*seconds),- *size,- ),- )- })- .collect()- }-- #[test]- fn changed_artifacts_detects_new_and_modified_beams() {- let before = ArtifactState {- beams: file_state(&[("ebin/a.beam", 1, 10), ("ebin/b.beam", 1, 20)]),- nifs: FileState::new(),- };- let after = ArtifactState {- beams: file_state(&[- ("ebin/a.beam", 2, 10),- ("ebin/b.beam", 1, 20),- ("ebin/c.beam", 1, 30),- ]),- nifs: FileState::new(),- };- let diff = changed_artifacts(&before, &after);- assert_eq!(diff.modules, vec!["a".to_owned(), "c".to_owned()]);- assert!(!diff.nifs_changed);- }-- #[test]- fn changed_artifacts_ignores_unchanged_state() {- let state = ArtifactState {- beams: file_state(&[("ebin/a.beam", 1, 10)]),- nifs: file_state(&[("priv/a_nif.so", 1, 10)]),- };- let diff = changed_artifacts(&state, &state.clone());- assert!(diff.modules.is_empty());- assert!(!diff.nifs_changed);- }-- #[test]- fn changed_artifacts_flags_nif_changes() {- let before = ArtifactState {- beams: FileState::new(),- nifs: file_state(&[("priv/a_nif.so", 1, 10)]),- };- let after = ArtifactState {- beams: FileState::new(),- nifs: file_state(&[("priv/a_nif.so", 2, 11)]),- };- assert!(changed_artifacts(&before, &after).nifs_changed);- }-- #[test]- fn reload_eval_quotes_nodes_and_modules() {- let eval = build_reload_eval(- &["[email protected]".to_owned()],- &["gateway_compress".to_owned(), "push".to_owned()],- );- assert!(eval.contains("Nodes = ['[email protected]']"));- assert!(eval.contains("Mods = ['gateway_compress','push']"));- assert!(eval.contains("halt(0)."));- }-- #[test]- fn reload_outcome_parses_failed_nodes() {- let output = "gateway_reload_error n1 mod {error,nofile}\n\- gateway_reload_failed_node [email protected]\n\- gateway_reload_failed_node [email protected]\n\- gateway_reload_done\n";- let outcome = parse_reload_outcome(output);- assert_eq!(- outcome.failed_nodes,- vec!["[email protected]".to_owned()]- );- }-- #[test]- fn reload_outcome_empty_when_no_failures() {- assert!(- parse_reload_outcome("gateway_reload_done\n")- .failed_nodes- .is_empty()- );- }-- #[test]- fn watched_source_filter_accepts_known_extensions() {- assert!(is_watched_source(Path::new("src/push.erl")));- assert!(is_watched_source(Path::new("include/gateway.hrl")));- assert!(is_watched_source(Path::new("src/fluxer_gateway.app.src")));- assert!(is_watched_source(Path::new("native/a_nif/src/lib.rs")));- assert!(is_watched_source(Path::new("native/a_nif/Cargo.toml")));- assert!(!is_watched_source(Path::new("ebin/push.beam")));- assert!(!is_watched_source(Path::new("src/.push.erl.swp")));- assert!(!is_watched_source(Path::new("native/a_nif/Cargo.lock")));- }-- #[test]- fn skipped_dirs_cover_build_output_and_hidden_dirs() {- assert!(is_skipped_dir(Path::new("native/a_nif/target")));- assert!(is_skipped_dir(Path::new("src/.git")));- assert!(!is_skipped_dir(Path::new("src/gateway")));- }-}diff --git a/tools/dev/src/lib.rs b/tools/dev/src/lib.rsindex 11f54c421..32122d28b 100644--- a/tools/dev/src/lib.rs+++ b/tools/dev/src/lib.rs@@ -7,7 +7,6 @@ pub mod dev;pub mod disclaim;pub mod env;pub mod gateway;-pub mod gateway_reload;pub mod manifest;pub mod media_external;pub mod media_proxy;diff --git a/tools/dev/src/main.rs b/tools/dev/src/main.rsindex 822e6695a..4f3f0a2ca 100644--- a/tools/dev/src/main.rs+++ b/tools/dev/src/main.rs@@ -1,6 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use anyhow::Result;+use anyhow::{Context, Result, bail};use clap::{Args, Parser, Subcommand};use fluxer_dev::cassandra::{apply_schema, compute_diff, render_target_schema, verify_schema, write_diff_file,@@ -11,9 +11,19 @@ use fluxer_dev::desktop::{};use fluxer_dev::env::merge_default_env_with_current;use fluxer_dev::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL};-use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT_LOCAL_ENV_FILE};+use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};use std::collections::BTreeMap;-use std::path::PathBuf;+use std::path::{Path, PathBuf};+use std::process::Command as ProcessCommand;++const DEV_INFRA_SERVICES: &[&str] = &[+ "postgres",+ "valkey",+ "nats",+ "livekit",+ "meilisearch",+ "mailpit",+];#[derive(Debug, Parser)]#[command(name = "fluxer-dev")]@@ -34,6 +44,7 @@ enum Command {Proxy(ProxyArgs),Dev(DevArgs),RustServices(RustServicesArgs),+ Infra(InfraArgs),Cassandra(CassandraArgs),Desktop(DesktopArgs),MediaProxy(MediaProxyArgs),@@ -44,8 +55,6 @@ enum Command {struct BootstrapArgs {#[arg(long)]skip_install: bool,- #[arg(long)]- skip_desktop_install: bool,}#[derive(Debug, Args)]@@ -76,6 +85,19 @@ struct RustServicesArgs {services: Vec<String>,}+#[derive(Debug, Args)]+struct InfraArgs {+ #[command(subcommand)]+ command: InfraCommand,+}++#[derive(Debug, Subcommand)]+enum InfraCommand {+ Start,+ Stop,+ Status,+}+#[derive(Debug, Args)]struct CassandraArgs {#[command(subcommand)]@@ -193,10 +215,12 @@ async fn main() -> Result<()> {match cli.command {Command::Bootstrap(args) => {- fluxer_dev::bootstrap::bootstrap(args.skip_install, args.skip_desktop_install).await?;+ fluxer_dev::bootstrap::bootstrap(args.skip_install).await?;}Command::PostStart => fluxer_dev::bootstrap::post_start().await?,- Command::Gateway(args) if args.mode == "single" => fluxer_dev::gateway::run_gateway()?,+ Command::Gateway(args) if args.mode == "single" => {+ std::process::exit(fluxer_dev::gateway::run_gateway().await?)+ }Command::Gateway(_) => {std::process::exit(fluxer_dev::gateway::run_gateway_cluster().await?)}@@ -216,6 +240,7 @@ async fn main() -> Result<()> {Command::RustServices(args) => {std::process::exit(fluxer_dev::rust_services::run_rust_services(&args.services).await?)}+ Command::Infra(args) => run_infra(args.command)?,Command::Cassandra(args) => match args.command {CassandraCommand::Diff { output } => {let diff = compute_diff(None).await?;@@ -315,3 +340,78 @@ fn apply_default_env() -> Result<()> {}Ok(())}++fn run_infra(command: InfraCommand) -> Result<()> {+ let project = compose_project_name()?;+ let compose_file = ROOT.join(".devcontainer/docker-compose.yml");+ let mut args = vec![+ "compose".to_owned(),+ "--project-name".to_owned(),+ project,+ "-f".to_owned(),+ compose_file.display().to_string(),+ ];+ match command {+ InfraCommand::Start => args.push("start".to_owned()),+ InfraCommand::Stop => args.push("stop".to_owned()),+ InfraCommand::Status => {+ args.push("ps".to_owned());+ args.push("--all".to_owned());+ }+ }+ args.extend(+ DEV_INFRA_SERVICES+ .iter()+ .map(|service| (*service).to_owned()),+ );+ let status = ProcessCommand::new("docker")+ .args(&args)+ .status()+ .context("failed to run Docker Compose for the dev infrastructure")?;+ if !status.success() {+ bail!("Docker Compose dev infrastructure command failed with {status}");+ }+ Ok(())+}++fn compose_project_name() -> Result<String> {+ if Path::new("/.dockerenv").exists() {+ let container = std::fs::read_to_string("/etc/hostname")+ .context("failed to read the current devcontainer hostname")?;+ let container = container.trim();+ if container.is_empty()+ || !container+ .bytes()+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-'))+ {+ bail!("Current devcontainer hostname is invalid");+ }+ let output = ProcessCommand::new("docker")+ .args([+ "inspect",+ "--format",+ "{{ index .Config.Labels \"com.docker.compose.project\" }}",+ container,+ ])+ .output()+ .context("failed to inspect the current devcontainer Compose project")?;+ if !output.status.success() {+ bail!(+ "Could not discover the current devcontainer Compose project: {}",+ String::from_utf8_lossy(&output.stderr).trim()+ );+ }+ let project = String::from_utf8(output.stdout)+ .context("devcontainer Compose project label is not valid UTF-8")?+ .trim()+ .to_owned();+ if project.is_empty() || project == "<no value>" {+ bail!("Current container has no com.docker.compose.project label");+ }+ return Ok(project);+ }+ std::env::var("COMPOSE_PROJECT_NAME")+ .ok()+ .filter(|project| !project.trim().is_empty())+ .context("COMPOSE_PROJECT_NAME must be set when managing dev infrastructure from the host")+}diff --git a/tools/dev/src/manifest.rs b/tools/dev/src/manifest.rsindex 99d0a613c..7dbae7836 100644--- a/tools/dev/src/manifest.rs+++ b/tools/dev/src/manifest.rs@@ -7,6 +7,7 @@ pub const LOOPBACK_HOST: &str = "127.0.0.1";pub const ANY_HOST: &str = "0.0.0.0";pub const DEV_PROXY_PORT: u16 = 8088;+pub const DEV_PROXY_GATEWAY_PORTS_ENV: &str = "FLUXER_DEV_PROXY_GATEWAY_PORTS";pub const APP_PORT: u16 = 3000;pub const APP_PROXY_PORT: u16 = 8773;pub const ADMIN_PORT: u16 = 3020;diff --git a/tools/dev/src/media_proxy.rs b/tools/dev/src/media_proxy.rsindex 414303011..e8dc3c50f 100644--- a/tools/dev/src/media_proxy.rs+++ b/tools/dev/src/media_proxy.rs@@ -17,6 +17,8 @@ const DEV_S3_ACCESS_KEY_ID: &str = "fluxer";const DEV_S3_SECRET_ACCESS_KEY: &str = "fluxer-secret";const DEV_S3_HOST: &str = "127.0.0.1";const DEV_S3_PORT: u16 = 8333;+const DEV_SEAWEEDFS_STOP_TIMEOUT: Duration = Duration::from_secs(15);+const DEV_SEAWEEDFS_KILL_TIMEOUT: Duration = Duration::from_secs(2);pub async fn run_dev_media_doctor(repair: bool,@@ -35,10 +37,20 @@ pub async fn run_dev_media_doctor(}pub async fn ensure_dev_object_store(repair: bool, repair_timeout_secs: u64) -> Result<()> {+ let managed_process_running =+ read_dev_seaweedfs_pid()?.is_some_and(managed_seaweedfs_process_running);+ if repair && !tcp_reachable(DEV_S3_HOST, DEV_S3_PORT) && !managed_process_running {+ println!("SeaweedFS S3 is not running; starting the managed development instance.");+ start_dev_seaweedfs()?;+ wait_s3_api(repair_timeout_secs).await?;+ ensure_s3_buckets()?;+ return Ok(());+ }+match wait_s3_api(5).await {Ok(()) => {}- Err(error) if repair => {- println!("SeaweedFS S3 check failed: {error}");+ Err(_) if repair => {+ println!("SeaweedFS S3 is unresponsive; restarting the managed development instance.");start_dev_seaweedfs()?;wait_s3_api(repair_timeout_secs).await?;}@@ -128,7 +140,7 @@ fn start_dev_seaweedfs() -> Result<()> {if let Some(pid) = read_dev_seaweedfs_pid()? {if managed_seaweedfs_process_running(pid) {eprintln!("Stopping unresponsive managed SeaweedFS process {pid}.");- stop_managed_seaweedfs_process(pid);+ stop_managed_seaweedfs_process(pid)?;}let _ = fs::remove_file(DEV_SEAWEEDFS_PID_FILE.as_path());}@@ -141,15 +153,24 @@ fn start_dev_seaweedfs() -> Result<()> {let log_path = DEV_LOG_DIR.join("seaweedfs.log");let log = OpenOptions::new().create(true)- .append(true)+ .write(true)+ .truncate(true).open(&log_path).with_context(|| format!("failed to open SeaweedFS log {}", log_path.display()))?;let stderr = log.try_clone().context("failed to clone SeaweedFS log handle")?;let data_dir_arg = format!("-dir={}", DEV_SEAWEEDFS_DIR.display());- let child = Command::new(weed)- .args(["-logtostderr=true", "mini", &data_dir_arg])+ let mut child = Command::new(weed)+ .args([+ "-logtostderr=true",+ "mini",+ &data_dir_arg,+ "-admin.ui=false",+ "-filer.disableDirListing",+ "-s3.port.iceberg=0",+ "-webdav=false",+ ]).env("AWS_ACCESS_KEY_ID", DEV_S3_ACCESS_KEY_ID).env("AWS_SECRET_ACCESS_KEY", DEV_S3_SECRET_ACCESS_KEY).env("S3_BUCKET", DEV_S3_BUCKETS)@@ -159,12 +180,16 @@ fn start_dev_seaweedfs() -> Result<()> {.spawn().context("failed to start SeaweedFS")?;let pid = child.id();- fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()).with_context(|| {- format!(- "failed to write SeaweedFS pid file {}",- DEV_SEAWEEDFS_PID_FILE.display()- )- })?;+ if let Err(error) = fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()) {+ let _ = child.kill();+ let _ = child.wait();+ return Err(error).with_context(|| {+ format!(+ "failed to write SeaweedFS pid file {}",+ DEV_SEAWEEDFS_PID_FILE.display()+ )+ });+ }println!("Started SeaweedFS dev object store with pid {pid}; logs: {}",log_path.display()@@ -195,20 +220,44 @@ fn managed_seaweedfs_process_running(pid: u32) -> bool {}#[cfg(unix)]-fn stop_managed_seaweedfs_process(pid: u32) {- unsafe {- libc::kill(pid as i32, libc::SIGTERM);+fn stop_managed_seaweedfs_process(pid: u32) -> Result<()> {+ let result = unsafe { libc::kill(pid as i32, libc::SIGTERM) };+ if result == -1 {+ let error = std::io::Error::last_os_error();+ if error.raw_os_error() != Some(libc::ESRCH) {+ return Err(error).context("failed to terminate managed SeaweedFS process");+ }}- std::thread::sleep(Duration::from_secs(2));- if managed_seaweedfs_process_running(pid) {- unsafe {- libc::kill(pid as i32, libc::SIGKILL);++ let terminate_deadline = Instant::now() + DEV_SEAWEEDFS_STOP_TIMEOUT;+ while managed_seaweedfs_process_running(pid) && Instant::now() < terminate_deadline {+ std::thread::sleep(Duration::from_millis(100));+ }+ if !managed_seaweedfs_process_running(pid) {+ return Ok(());+ }++ let result = unsafe { libc::kill(pid as i32, libc::SIGKILL) };+ if result == -1 {+ let error = std::io::Error::last_os_error();+ if error.raw_os_error() != Some(libc::ESRCH) {+ return Err(error).context("failed to kill unresponsive managed SeaweedFS process");}}+ let kill_deadline = Instant::now() + DEV_SEAWEEDFS_KILL_TIMEOUT;+ while managed_seaweedfs_process_running(pid) && Instant::now() < kill_deadline {+ std::thread::sleep(Duration::from_millis(100));+ }+ if managed_seaweedfs_process_running(pid) {+ bail!("managed SeaweedFS process {pid} survived SIGKILL");+ }+ Ok(())}#[cfg(not(unix))]-fn stop_managed_seaweedfs_process(_pid: u32) {}+fn stop_managed_seaweedfs_process(_pid: u32) -> Result<()> {+ Ok(())+}async fn check_dev_media_path(base_url: &str, path: &str) -> Result<()> {let path = if path.starts_with('/') {diff --git a/tools/dev/src/object_store.rs b/tools/dev/src/object_store.rsindex 4c9ee06a9..69bc4810e 100644--- a/tools/dev/src/object_store.rs+++ b/tools/dev/src/object_store.rs@@ -168,11 +168,11 @@ fn s3_bucket_already_exists_output(output: &str) -> bool {output.contains("BucketAlreadyExists") || output.contains("BucketAlreadyOwnedByYou")}-pub async fn bootstrap_schema_and_object_store() -> Result<()> {+pub async fn bootstrap_schema() -> Result<()> {if cassandra_backend_enabled() {apply_schema(Some(config_from_env()?)).await?;}- ensure_s3_buckets()+ Ok(())}fn cassandra_backend_enabled() -> bool {diff --git a/tools/dev/src/proc.rs b/tools/dev/src/proc.rsindex 95282721e..6c5b35d00 100644--- a/tools/dev/src/proc.rs+++ b/tools/dev/src/proc.rs@@ -4,9 +4,11 @@ use crate::env::merge_default_env_with_current;use crate::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};use anyhow::{Context, Result, bail};use std::collections::{BTreeMap, VecDeque};+use std::future::Future;+use std::io::{BufRead, BufReader, Read};use std::net::{TcpStream, ToSocketAddrs};use std::path::Path;-use std::process::{Command, Output, Stdio};+use std::process::{Child, Command, Output, Stdio};use std::time::{Duration, Instant};use tokio::time::sleep;@@ -172,6 +174,152 @@ pub fn run_command(args: &[&str], options: RunOptions<'_>) -> Result<Output> {Ok(output)}+pub async fn run_command_interruptible(+ args: &[&str],+ options: RunOptions<'_>,+ shutdown: &mut ShutdownSignal,+) -> Result<AwaitOutcome<Output>> {+ if options.capture {+ bail!("interruptible commands do not support captured output");+ }+ println!("$ {}", format_command(args));+ let env = merged_env(Some(&options.env), options.load_default_env)?;+ let mut command = Command::new(args[0]);+ command+ .args(&args[1..])+ .current_dir(options.cwd)+ .env_clear()+ .envs(env)+ .stdin(Stdio::inherit())+ .stdout(Stdio::inherit())+ .stderr(Stdio::inherit());+ configure_process_group(&mut command);+ let mut child = command+ .spawn()+ .with_context(|| format!("failed to run {}", format_command(args)))?;+ loop {+ let status = match child.try_wait() {+ Ok(status) => status,+ Err(error) => {+ stop_process_group(&mut child, Duration::from_secs(5)).await;+ return Err(error)+ .with_context(|| format!("failed to wait for {}", format_command(args)));+ }+ };+ if let Some(status) = status {+ stop_process_group(&mut child, Duration::from_secs(5)).await;+ let output = Output {+ status,+ stdout: Vec::new(),+ stderr: Vec::new(),+ };+ if options.check && !output.status.success() {+ let code = output.status.code().unwrap_or(-1);+ bail!(+ "Command failed with exit code {code}: {}",+ format_command(args)+ );+ }+ return Ok(AwaitOutcome::Completed(output));+ }+ tokio::select! {+ signal = shutdown.recv() => {+ stop_process_group(&mut child, Duration::from_secs(5)).await;+ return Ok(AwaitOutcome::Shutdown(signal));+ }+ _ = sleep(Duration::from_millis(100)) => {}+ }+ }+}++pub fn configure_process_group(command: &mut Command) {+ #[cfg(unix)]+ unsafe {+ use std::os::unix::process::CommandExt;+ command.pre_exec(|| {+ if libc::setsid() == -1 {+ return Err(std::io::Error::last_os_error());+ }+ Ok(())+ });+ }+}++#[cfg(unix)]+pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> {+ signal_process_group(process, libc::SIGTERM)+}++#[cfg(not(unix))]+pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> {+ process.kill()+}++#[cfg(unix)]+pub fn force_kill_process_group(process: &mut Child) {+ if let Err(error) = signal_process_group(process, libc::SIGKILL) {+ eprintln!(+ "Failed to send SIGKILL to process group {}: {error}",+ process.id()+ );+ }+ let _ = process.kill();+ let _ = process.wait();+}++#[cfg(unix)]+pub fn process_group_running(process: &mut Child) -> bool {+ if unsafe { libc::kill(-(process.id() as i32), 0) } == 0 {+ return true;+ }+ let error = std::io::Error::last_os_error();+ error.raw_os_error() != Some(libc::ESRCH)+}++#[cfg(not(unix))]+pub fn process_group_running(process: &mut Child) -> bool {+ process.try_wait().ok().flatten().is_none()+}++#[cfg(not(unix))]+pub fn force_kill_process_group(process: &mut Child) {+ let _ = process.kill();+ let _ = process.wait();+}++#[cfg(unix)]+fn signal_process_group(process: &Child, signal: libc::c_int) -> std::io::Result<()> {+ if unsafe { libc::kill(-(process.id() as i32), signal) } == 0 {+ return Ok(());+ }+ let error = std::io::Error::last_os_error();+ if error.raw_os_error() == Some(libc::ESRCH) {+ return Ok(());+ }+ Err(error)+}++pub async fn stop_process_group(process: &mut Child, grace_period: Duration) {+ if let Err(error) = terminate_process_group(process) {+ eprintln!(+ "Failed to terminate process group {}: {error}",+ process.id()+ );+ }+ let deadline = Instant::now() + grace_period;+ loop {+ let leader_exited = process.try_wait().ok().flatten().is_some();+ if leader_exited && !process_group_running(process) {+ return;+ }+ if Instant::now() >= deadline {+ force_kill_process_group(process);+ return;+ }+ sleep(Duration::from_millis(100)).await;+ }+}+pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> Result<()> {let deadline = Instant::now() + Duration::from_secs(timeout_secs);let mut last_error = None;@@ -199,6 +347,19 @@ pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> R}pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {+ wait_http_status(name, url, timeout_secs, |status| status.as_u16() < 500).await+}++pub async fn wait_http_success(name: &str, url: &str, timeout_secs: u64) -> Result<()> {+ wait_http_status(name, url, timeout_secs, |status| status.is_success()).await+}++async fn wait_http_status(+ name: &str,+ url: &str,+ timeout_secs: u64,+ accepts: impl Fn(reqwest::StatusCode) -> bool,+) -> Result<()> {let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build()?;@@ -206,7 +367,7 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {let mut last_error = None;while Instant::now() < deadline {match client.get(url).send().await {- Ok(response) if response.status().as_u16() < 500 => {+ Ok(response) if accepts(response.status()) => {println!("{name} is reachable at {url}");return Ok(());}@@ -221,6 +382,30 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {);}+pub fn prefix_output(label: &str, reader: impl Read) {+ let mut reader = BufReader::new(reader);+ let mut line = Vec::new();+ loop {+ line.clear();+ match reader.read_until(b'\n', &mut line) {+ Ok(0) => return,+ Ok(_) => {+ if line.last() == Some(&b'\n') {+ line.pop();+ }+ if line.last() == Some(&b'\r') {+ line.pop();+ }+ println!("[{label}] {}", String::from_utf8_lossy(&line));+ }+ Err(error) => {+ eprintln!("[{label}] output reader failed: {error}");+ return;+ }+ }+ }+}+pub const RESTART_WINDOW: Duration = Duration::from_secs(60);pub const RESTART_LIMIT: usize = 5;@@ -262,6 +447,29 @@ impl ShutdownSignal {}}+pub enum AwaitOutcome<T> {+ Completed(T),+ Shutdown(&'static str),+}++pub async fn await_or_shutdown<T, F>(shutdown: &mut ShutdownSignal, future: F) -> AwaitOutcome<T>+where+ F: Future<Output = T>,+{+ tokio::select! {+ biased;+ signal = shutdown.recv() => AwaitOutcome::Shutdown(signal),+ output = future => AwaitOutcome::Completed(output),+ }+}++pub async fn pending_shutdown(shutdown: &mut ShutdownSignal) -> Option<&'static str> {+ match await_or_shutdown(shutdown, std::future::ready(())).await {+ AwaitOutcome::Completed(()) => None,+ AwaitOutcome::Shutdown(signal) => Some(signal),+ }+}+#[cfg(not(unix))]pub struct ShutdownSignal;diff --git a/tools/dev/src/proxy.rs b/tools/dev/src/proxy.rsindex 0920f4c11..5f88001ee 100644--- a/tools/dev/src/proxy.rs+++ b/tools/dev/src/proxy.rs@@ -1,6 +1,9 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use crate::manifest::{ANY_HOST, DEV_PROXY_PORT, LOCAL_APP_URL, PROXY_ROUTES, ProxyRoute};+use crate::manifest::{+ ANY_HOST, DEV_PROXY_GATEWAY_PORTS_ENV, DEV_PROXY_PORT, GATEWAY_PORT, LOCAL_APP_URL,+ PROXY_ROUTES, ProxyRoute,+};use anyhow::{Context, Result, bail};use axum::{Router,@@ -14,7 +17,7 @@ use hyper_util::rt::TokioIo;use std::collections::HashMap;use std::env;use std::net::SocketAddr;-use std::sync::{LazyLock, Mutex};+use std::sync::{Arc, LazyLock, Mutex};use std::time::Duration;use tokio::io::{AsyncReadExt, AsyncWriteExt};use tokio::net::{TcpListener, TcpStream};@@ -26,6 +29,7 @@ static ROUTE_CURSORS: LazyLock<Mutex<HashMap<&'static str, usize>>> =#[derive(Clone)]struct ProxyState {http_client: reqwest::Client,+ gateway_ports: Arc<[u16]>,}const BLOCKED_REQUEST_HEADERS: &[&str] = &[@@ -67,9 +71,21 @@ pub async fn run_proxy(host: &str, port: u16) -> Result<()> {.tcp_nodelay(true).build().context("failed to build dev proxy HTTP client")?;+ let gateway_ports = gateway_proxy_ports()?;+ println!(+ "Fluxer dev proxy gateway ports: {}",+ gateway_ports+ .iter()+ .map(u16::to_string)+ .collect::<Vec<_>>()+ .join(",")+ );let app = Router::new().fallback(any(proxy_request))- .with_state(ProxyState { http_client });+ .with_state(ProxyState {+ http_client,+ gateway_ports,+ });axum::serve(listener,@@ -91,11 +107,29 @@ async fn proxy_request(}let route = route_for_path(&request_head.path);+ let (target_host, target_port) = target_for_route(route, &state.gateway_ports);if is_upgrade_request(&request_head) {- return proxy_upgrade(request, request_head, route, client_addr).await;+ return proxy_upgrade(+ request,+ request_head,+ route,+ target_host,+ target_port,+ client_addr,+ )+ .await;}- proxy_http(state, request, request_head, route, client_addr).await+ proxy_http(+ state,+ request,+ request_head,+ route,+ target_host,+ target_port,+ client_addr,+ )+ .await}async fn proxy_http(@@ -103,9 +137,10 @@ async fn proxy_http(request: Request<Body>,request_head: RequestHead,route: &'static ProxyRoute,+ target_host: &'static str,+ target_port: u16,client_addr: SocketAddr,) -> Response<Body> {- let (target_host, target_port) = target_for_route(route);let target_url = upstream_http_url(&request_head.path, route, target_host, target_port);let (parts, body) = request.into_parts();let method = parts.method.clone();@@ -155,10 +190,11 @@ async fn proxy_upgrade(mut request: Request<Body>,request_head: RequestHead,route: &'static ProxyRoute,+ target_host: &'static str,+ target_port: u16,client_addr: SocketAddr,) -> Response<Body> {let on_upgrade = hyper::upgrade::on(&mut request);- let (target_host, target_port) = target_for_route(route);let mut target = match TcpStream::connect((target_host, target_port)).await {Ok(target) => target,Err(error) => return bad_gateway_response(error),@@ -497,17 +533,59 @@ pub fn route_for_path(path: &str) -> &'static ProxyRoute {PROXY_ROUTES.last().expect("proxy has fallback route")}-pub fn target_for_route(route: &'static ProxyRoute) -> (&'static str, u16) {- if route.alternate_ports.is_empty() {+pub fn target_for_route(route: &'static ProxyRoute, gateway_ports: &[u16]) -> (&'static str, u16) {+ let uses_gateway_ports = route.prefix == "/gateway";+ let port_count = if uses_gateway_ports {+ gateway_ports.len()+ } else {+ route.alternate_ports.len() + 1+ };+ assert!(+ port_count > 0,+ "proxy route must have at least one target port"+ );+ if port_count == 1 {+ if uses_gateway_ports {+ return (route.host, gateway_ports[0]);+ }return (route.host, route.port);}- let ports = std::iter::once(route.port)- .chain(route.alternate_ports.iter().copied())- .collect::<Vec<_>>();let mut cursors = ROUTE_CURSORS.lock().expect("route cursor lock poisoned");let cursor = *cursors.get(route.prefix).unwrap_or(&0);- cursors.insert(route.prefix, (cursor + 1) % ports.len());- (route.host, ports[cursor])+ cursors.insert(route.prefix, (cursor + 1) % port_count);+ let port = if uses_gateway_ports {+ gateway_ports[cursor]+ } else if cursor == 0 {+ route.port+ } else {+ route.alternate_ports[cursor - 1]+ };+ (route.host, port)+}++fn gateway_proxy_ports() -> Result<Arc<[u16]>> {+ let raw = env::var(DEV_PROXY_GATEWAY_PORTS_ENV).unwrap_or_else(|_| GATEWAY_PORT.to_string());+ let mut ports = Vec::new();+ for token in raw.split(',') {+ let token = token.trim();+ if token.is_empty() {+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains an empty port");+ }+ let port = token+ .parse::<u16>()+ .with_context(|| format!("Invalid port {token:?} in {DEV_PROXY_GATEWAY_PORTS_ENV}"))?;+ if port == 0 {+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} ports must be greater than zero");+ }+ if ports.contains(&port) {+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains duplicate port {port}");+ }+ ports.push(port);+ }+ if ports.len() > 3 {+ bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} supports at most three ports");+ }+ Ok(ports.into())}pub fn rewrite_path(path: &str, route: &ProxyRoute) -> String {diff --git a/tools/dev/src/rust_services.rs b/tools/dev/src/rust_services.rsindex e32bcc00d..33b1c3099 100644--- a/tools/dev/src/rust_services.rs+++ b/tools/dev/src/rust_services.rs@@ -1,10 +1,11 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse crate::manifest::{DEV_PROXY_PORT, MEDIA_PROXY_PORT, RustServiceSpec, rust_services};-use crate::paths::ROOT;+use crate::paths::{ROOT, TARGET_DIR};use crate::proc::{- RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,- restart_budget_exceeded, run_command,+ AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown,+ configure_process_group, format_command, merged_env, pending_shutdown, prefix_output,+ restart_budget_exceeded, run_command_interruptible,};use anyhow::{Result, bail};use std::collections::{BTreeSet, VecDeque};@@ -12,6 +13,8 @@ use std::env;#[cfg(target_os = "linux")]use std::fs;use std::net::{SocketAddr, TcpListener};+#[cfg(target_os = "linux")]+use std::path::Path;use std::process::{Child, Command, Stdio};use std::time::{Duration, Instant};use tokio::time::sleep;@@ -34,26 +37,51 @@ impl SupervisedService {pub async fn run_rust_services(service_names: &[String]) -> Result<i32> {let selected = select_services(service_names)?;+ let mut shutdown = ShutdownSignal::new()?;cleanup_orphaned_service_processes(&selected).await?;+ if let Some(signal) = pending_shutdown(&mut shutdown).await {+ println!("Received {signal}; stopping Rust service startup...");+ return Ok(0);+ }wait_for_service_ports_available(&selected)?;- build_services(&selected)?;- let mut shutdown = ShutdownSignal::new()?;+ match build_services(&selected, &mut shutdown).await? {+ AwaitOutcome::Completed(_) => {}+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping Rust service startup...");+ return Ok(0);+ }+ }let mut supervised = Vec::new();for spec in &selected {for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {+ let child = match start_service(spec, mode, port) {+ Ok(child) => child,+ Err(error) => {+ stop_supervised_services(&mut supervised);+ return Err(error);+ }+ };supervised.push(SupervisedService {spec: spec.clone(),mode,port,- child: start_service(spec, mode, port)?,+ child,restarts: VecDeque::new(),});}}loop {- if let Err(error) = restart_exited_services(&mut supervised).await {- stop_supervised_services(&mut supervised);- return Err(error);+ match await_or_shutdown(&mut shutdown, restart_exited_services(&mut supervised)).await {+ AwaitOutcome::Completed(Ok(())) => {}+ AwaitOutcome::Completed(Err(error)) => {+ stop_supervised_services(&mut supervised);+ return Err(error);+ }+ AwaitOutcome::Shutdown(signal) => {+ println!("Received {signal}; stopping Rust service tasks...");+ stop_supervised_services(&mut supervised);+ return Ok(0);+ }}tokio::select! {signal = shutdown.recv() => {@@ -71,6 +99,7 @@ async fn restart_exited_services(supervised: &mut [SupervisedService]) -> Resultlet Some(status) = entry.child.try_wait()? else {continue;};+ crate::gateway::stop_child_processes(&mut [&mut entry.child]);if restart_budget_exceeded(&mut entry.restarts, Instant::now()) {bail!("Rust service {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",@@ -145,7 +174,10 @@ pub fn select_services(service_names: &[String]) -> Result<Vec<RustServiceSpec>>.collect())}-fn build_services(services: &[RustServiceSpec]) -> Result<()> {+async fn build_services(+ services: &[RustServiceSpec],+ shutdown: &mut ShutdownSignal,+) -> Result<AwaitOutcome<std::process::Output>> {let mut packages = Vec::new();let mut seen = BTreeSet::new();for spec in services {@@ -156,7 +188,23 @@ fn build_services(services: &[RustServiceSpec]) -> Result<()> {let mut args = vec!["cargo".to_owned(), "build".to_owned()];args.extend(packages);let refs = args.iter().map(String::as_str).collect::<Vec<_>>();- run_command(&refs, RunOptions::default()).map(drop)+ let env = if env::var_os("CARGO_BUILD_JOBS").is_none() {+ vec![(+ "CARGO_BUILD_JOBS".to_owned(),+ Some(env::var("FLUXER_DEV_CARGO_JOBS").unwrap_or_else(|_| "2".to_owned())),+ )]+ } else {+ Vec::new()+ };+ run_command_interruptible(+ &refs,+ RunOptions {+ env,+ ..RunOptions::default()+ },+ shutdown,+ )+ .await}fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child> {@@ -172,14 +220,7 @@ fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child>.envs(env).stdout(Stdio::piped()).stderr(Stdio::piped());- #[cfg(unix)]- unsafe {- use std::os::unix::process::CommandExt;- command.pre_exec(|| {- libc::setsid();- Ok(())- });- }+ configure_process_group(&mut command);let mut child = command.spawn()?;if let Some(stdout) = child.stdout.take() {let stdout_label = label.clone();@@ -212,11 +253,20 @@ pub fn service_command(spec: &RustServiceSpec) -> Vec<String> {format!("run -p {}", spec.package),];}+ let target_dir = env::var_os("CARGO_TARGET_DIR")+ .map(std::path::PathBuf::from)+ .unwrap_or_else(|| TARGET_DIR.clone());+ let target_dir = if target_dir.is_absolute() {+ target_dir+ } else {+ ROOT.join(target_dir)+ };vec![- "cargo".to_owned(),- "run".to_owned(),- "-p".to_owned(),- spec.package.to_owned(),+ target_dir+ .join("debug")+ .join(format!("{}{}", spec.package, std::env::consts::EXE_SUFFIX))+ .display()+ .to_string(),]}@@ -238,18 +288,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String.unwrap_or_else(|_| "nats://nats:4222".to_owned()),),),- (- "FLUXER_CASSANDRA_HOSTS".to_owned(),- Some(env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned())),- ),- (- "FLUXER_CASSANDRA_KEYSPACE".to_owned(),- Some(env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned())),- ),- (- "FLUXER_CASSANDRA_PORT".to_owned(),- Some(env::var("FLUXER_CASSANDRA_PORT").unwrap_or_else(|_| "9042".to_owned())),- ),];if mode == "shard" {envs.push(("FLUXER_SVC_SHARD_ID".to_owned(), Some("0".to_owned())));@@ -284,13 +322,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(Stringenvs}-fn prefix_output(label: &str, reader: impl std::io::Read) {- use std::io::{BufRead, BufReader};- for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {- println!("[{label}] {line}");- }-}-fn wait_for_service_ports_available(services: &[RustServiceSpec]) -> Result<()> {let conflicts = services.iter()@@ -333,7 +364,7 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Res"Stopping orphaned Rust service process group(s): {}",format_pids(&pids));- let term_failed = signal_process_groups(&pids, libc::SIGTERM);+ let term_failed = signal_process_groups(&leaders, libc::SIGTERM);if !term_failed.is_empty() {println!("SIGTERM delivery failed for orphaned Rust service pid(s): {}",@@ -348,7 +379,12 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Resreturn Ok(());}if Instant::now() >= deadline {- let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);+ let remaining_leaders = leaders+ .iter()+ .filter(|leader| remaining.contains(&leader.pid))+ .cloned()+ .collect::<Vec<_>>();+ let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL);sleep(Duration::from_millis(200)).await;let survivors = surviving_service_group_pids(&leaders)?;if survivors.is_empty() {@@ -380,19 +416,85 @@ async fn cleanup_orphaned_service_processes(_services: &[RustServiceSpec]) -> Re}#[cfg(target_os = "linux")]-#[derive(Clone, Copy, Debug, Eq, PartialEq)]+#[derive(Clone, Debug, Eq, PartialEq)]struct RustServiceLeader {+ pid: i32,+ members: Vec<RustServiceProcessIdentity>,+}++#[cfg(target_os = "linux")]+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+struct RustServiceProcessIdentity {pid: i32,starttime: u64,}+#[cfg(target_os = "linux")]+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+struct RustServiceProcess {+ pid: i32,+ ppid: i32,+ pgid: i32,+ state: char,+ starttime: u64,+}+#[cfg(target_os = "linux")]fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServiceLeader>> {- let binaries = services+ let current_exe = std::env::current_exe()?;+ let processes = service_process_snapshot()?;+ let mut owned_pids = BTreeSet::new();+ for process in &processes {+ if process.ppid != 1 || proc_stat_state_is_dead(process.state) {+ continue;+ }+ let args = proc_cmdline(process.pid);+ let orphaned_supervisor = cmdline_is_service_supervisor(&args, ¤t_exe);+ let orphaned_service = proc_has_managed_service_environment(process.pid, services)+ && fs::read_link(format!("/proc/{}/cwd", process.pid))+ .ok()+ .as_deref()+ == Some(ROOT.as_path());+ if orphaned_supervisor || orphaned_service {+ owned_pids.insert(process.pid);+ }+ }+ loop {+ let previous_count = owned_pids.len();+ for process in &processes {+ if owned_pids.contains(&process.ppid) {+ owned_pids.insert(process.pid);+ }+ }+ if owned_pids.len() == previous_count {+ break;+ }+ }+ let group_pids = processes.iter()- .map(|spec| format!("target/debug/{}", spec.package))+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1)+ .map(|process| process.pgid).collect::<BTreeSet<_>>();- let mut leaders = Vec::new();+ let leaders = group_pids+ .into_iter()+ .map(|pid| RustServiceLeader {+ pid,+ members: processes+ .iter()+ .filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid)+ .map(|process| RustServiceProcessIdentity {+ pid: process.pid,+ starttime: process.starttime,+ })+ .collect(),+ })+ .collect::<Vec<_>>();+ Ok(leaders)+}++#[cfg(target_os = "linux")]+fn service_process_snapshot() -> Result<Vec<RustServiceProcess>> {+ let mut processes = Vec::new();for entry in fs::read_dir("/proc")? {let Ok(entry) = entry else {continue;@@ -404,18 +506,69 @@ fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServelse {continue;};- if proc_parent_pid(pid) != Some(1) {+ let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {continue;- }- if !cmdline_has_service_binary(&proc_cmdline(pid), &binaries) {+ };+ let Some((state, ppid, pgid, starttime)) = parse_proc_stat_process(&stat) else {continue;- }- if let Some(starttime) = proc_stat_starttime(pid) {- leaders.push(RustServiceLeader { pid, starttime });- }+ };+ processes.push(RustServiceProcess {+ pid,+ ppid,+ pgid,+ state,+ starttime,+ });}- leaders.sort_unstable_by_key(|leader| leader.pid);- Ok(leaders)+ Ok(processes)+}++#[cfg(target_os = "linux")]+fn cmdline_is_service_supervisor(args: &[String], current_exe: &Path) -> bool {+ args.first().map(Path::new) == Some(current_exe)+ && args.get(1).map(String::as_str) == Some("rust-services")+}++#[cfg(target_os = "linux")]+fn proc_has_managed_service_environment(pid: i32, services: &[RustServiceSpec]) -> bool {+ let environment_bytes = fs::read(format!("/proc/{pid}/environ")).unwrap_or_default();+ let environment = environment_bytes+ .split(|byte| *byte == 0)+ .filter_map(|entry| {+ let separator = entry.iter().position(|byte| *byte == b'=')?;+ let (key, value) = entry.split_at(separator);+ Some((key, &value[1..]))+ })+ .filter_map(|(key, value)| {+ Some((+ std::str::from_utf8(key).ok()?,+ std::str::from_utf8(value).ok()?,+ ))+ })+ .collect::<std::collections::HashMap<_, _>>();+ let Some(name) = environment.get("FLUXER_SVC_NAME") else {+ return false;+ };+ let Some(mode) = environment.get("FLUXER_SVC_MODE") else {+ return false;+ };+ let Some(port) = environment+ .get("FLUXER_SVC_PORT")+ .and_then(|port| port.parse::<u16>().ok())+ else {+ return false;+ };+ if environment.get("FLUXER_SVC_LISTEN_HOST") != Some(&"0.0.0.0") {+ return false;+ }+ services.iter().any(|service| {+ service.name == *name+ && match *mode {+ "router" => port == service.port_base,+ "shard" => port == service.port_base + 1,+ _ => false,+ }+ })}#[cfg(target_os = "linux")]@@ -429,7 +582,7 @@ fn format_pids(pids: &[i32]) -> String {.join(", ")}-#[cfg(any(target_os = "linux", test))]+#[cfg(test)]fn cmdline_has_service_binary(args: &[String], binaries: &BTreeSet<String>) -> bool {args.first().map(|arg| binaries.iter().any(|binary| arg.ends_with(binary)))@@ -446,30 +599,6 @@ fn proc_cmdline(pid: i32) -> Vec<String> {.collect()}-#[cfg(target_os = "linux")]-fn proc_parent_pid(pid: i32) -> Option<i32> {- fs::read_to_string(format!("/proc/{pid}/status"))- .ok()?- .lines()- .find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())-}--#[cfg(target_os = "linux")]-fn process_exists(pid: i32) -> bool {- assert!(pid > 0);- match proc_stat_state_and_pgid(pid) {- Some((state, _pgid)) => !proc_stat_state_is_dead(state),- None => false,- }-}--#[cfg(target_os = "linux")]-fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {- assert!(pid > 0);- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;- parse_proc_stat_state_and_pgid(&stat)-}-#[cfg(any(target_os = "linux", test))]fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {let (_, after_comm) = stat.rsplit_once(')')?;@@ -481,10 +610,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {}#[cfg(target_os = "linux")]-fn proc_stat_starttime(pid: i32) -> Option<u64> {- assert!(pid > 0);- let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;- parse_proc_stat_starttime(&stat)+fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> {+ let (_, after_comm) = stat.rsplit_once(')')?;+ let mut fields = after_comm.split_ascii_whitespace();+ let state = fields.next()?.chars().next()?;+ let ppid = fields.next()?.parse().ok()?;+ let pgid = fields.next()?.parse().ok()?;+ Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?))}#[cfg(any(target_os = "linux", test))]@@ -500,62 +632,43 @@ fn proc_stat_state_is_dead(state: char) -> bool {#[cfg(target_os = "linux")]fn surviving_service_group_pids(leaders: &[RustServiceLeader]) -> Result<Vec<i32>> {- let leader_pids = leaders+ Ok(leaders.iter()+ .filter(|leader| service_group_has_owned_member(leader)).map(|leader| leader.pid)- .collect::<std::collections::HashSet<_>>();- let mut survivors = Vec::new();- for entry in fs::read_dir("/proc")? {- let Ok(entry) = entry else {- continue;- };- let Some(pid) = entry- .file_name()- .to_str()- .and_then(|name| name.parse::<i32>().ok())- else {- continue;- };- let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {- continue;+ .collect())+}++#[cfg(target_os = "linux")]+fn service_group_has_owned_member(leader: &RustServiceLeader) -> bool {+ leader.members.iter().any(|member| {+ let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else {+ return false;};let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {- continue;+ return false;};- if proc_stat_state_is_dead(state) {- continue;- }- if leader_pids.contains(&pgid) {- survivors.push(pid);- continue;- }- let leader = leaders.iter().find(|leader| leader.pid == pid);- if let Some(leader) = leader- && parse_proc_stat_starttime(&stat) == Some(leader.starttime)- {- survivors.push(pid);- }- }- survivors.sort_unstable();- Ok(survivors)+ !proc_stat_state_is_dead(state)+ && pgid == leader.pid+ && parse_proc_stat_starttime(&stat) == Some(member.starttime)+ })}#[cfg(target_os = "linux")]-fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {+fn signal_process_groups(leaders: &[RustServiceLeader], signal: i32) -> Vec<i32> {assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);- let mut failed = Vec::with_capacity(pids.len());- for pid in pids {- assert!(*pid > 0);- let group_result = unsafe { libc::kill(-pid, signal) };- if group_result == 0 {+ let mut failed = Vec::with_capacity(leaders.len());+ for leader in leaders {+ assert!(leader.pid > 1);+ if !service_group_has_owned_member(leader) {continue;}- let direct_result = unsafe { libc::kill(*pid, signal) };- if direct_result == 0 {+ let group_result = unsafe { libc::kill(-leader.pid, signal) };+ if group_result == 0 {continue;}- if process_exists(*pid) {- failed.push(*pid);+ if service_group_has_owned_member(leader) {+ failed.push(leader.pid);}}faileddiff --git a/tools/dev/src/tunnel.rs b/tools/dev/src/tunnel.rsindex 6b46840cd..22558f3ed 100644--- a/tools/dev/src/tunnel.rs+++ b/tools/dev/src/tunnel.rs@@ -253,6 +253,16 @@ pub async fn run_cloudflare_tunnel(return Ok(status.code().unwrap_or(1));}if running_inside_devcontainer() {+ let container = std::fs::read_to_string("/etc/hostname")+ .context("failed to read the current devcontainer hostname")?;+ let container = container.trim();+ if container.is_empty()+ || !container+ .bytes()+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-'))+ {+ bail!("Current devcontainer hostname is invalid");+ }let docker = docker_command();let mut command = Command::new(&docker[0]);command.args(&docker[1..]);@@ -261,11 +271,7 @@ pub async fn run_cloudflare_tunnel("run","--rm","--network",- &format!(- "container:{}",- std::env::var("HOSTNAME")- .unwrap_or_else(|_| "fluxer-dev-workspace-1".to_owned())- ),+ &format!("container:{container}"),"cloudflare/cloudflared:latest","tunnel","run",