Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

ci(scripts): real install+update e2e gate, nightly tarball/from-release runs

Juan Denis4874fa91614f
diff --git a/.github/workflows/scripts-ci.yml b/.github/workflows/scripts-ci.yml
index c9e7f506..c8000dcb 100644
--- a/.github/workflows/scripts-ci.yml
+++ b/.github/workflows/scripts-ci.yml
@@ -6,6 +6,11 @@ name: Scripts CI
# box. shellcheck catches that whole class statically; the test_*.sh suites
# exercise the install/update/uninstall + lib logic against fixtures with no
# server, on every supported distro family.
+#
+# The unit suites *source* the scripts, though — the top-level run block
+# (exactly where the 2026-07-02 update abort lived) executes in none of them.
+# The update-e2e job closes that gap with a REAL install + REAL update on the
+# runner VM, so the code path a user's box actually runs is gated on every PR.
on:
push:
@@ -64,11 +69,13 @@ jobs:
# quality doesn't rot. -x follows `source`d files.
shellcheck --severity=warning -x scripts/lib/*.sh
- - name: Unit tests (update + install + lib)
+ - name: Unit tests (update + install + lib + cli + agent install)
run: |
bash scripts/test/test_update.sh
bash scripts/test/test_install.sh
bash scripts/test/test_lib.sh
+ bash scripts/test/test_cli.sh
+ bash scripts/test/test_agent_install.sh
- name: nginx -t smoke (shipped vhosts parse on a real nginx)
run: |
@@ -123,4 +130,115 @@ jobs:
bash scripts/test/test_update.sh
bash scripts/test/test_install.sh
bash scripts/test/test_lib.sh
+ bash scripts/test/test_cli.sh
+ bash scripts/test/test_agent_install.sh
'
+
+ update-e2e:
+ name: e2e real install + update
+ # The unit suites source the scripts and test functions in isolation — the
+ # top-level run block never executes there, and that run block is exactly
+ # where the 2026-07-02 update abort lived. This job is the missing gate: a
+ # REAL install (the default non-interactive, no-domain, plain-HTTP path)
+ # followed by a REAL run of THIS branch's updater, on the runner VM (full
+ # systemd + docker — the same shape as a user's box). --dry-run is
+ # deliberately NOT used: dry-run returns early from the very functions that
+ # have broken. This job mutates the runner (/opt/serverkit, systemd units,
+ # nginx), so it must stay independent — never fold it into the unit job.
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install ServerKit (non-interactive, no domain, plain HTTP)
+ run: |
+ # `sudo env`, not `... | sudo bash`: sudo resets the environment, so
+ # the switches must ride on the env command. PATH is forwarded so the
+ # runner's preinstalled node/npm/docker toolchain stays visible to
+ # root. No PANEL_DOMAIN + non-tty stdin is the documented no-prompt
+ # default; SERVERKIT_SKIP_SSL keeps the run HTTP-only (no certbot
+ # attempt on a runner with no DNS). install.sh copes with whatever
+ # web-server state the runner image ships — it is built for real,
+ # already-populated boxes.
+ sudo env PATH="$PATH" SERVERKIT_SKIP_SSL=1 bash install.sh
+
+ - name: Panel healthy after install
+ run: |
+ # Same probe install.sh and the VM harness gate on.
+ for i in $(seq 1 30); do
+ curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
+ echo "backend healthy after install (probe $i)"; exit 0; }
+ sleep 2
+ done
+ echo "backend never became healthy after install" >&2
+ exit 1
+
+ - name: Real update with THIS branch's updater (--force)
+ run: |
+ # SERVERKIT_NO_SELF_UPDATE=1 stops the bootstrap from re-execing into
+ # the updater published on the target ref — the object under test is
+ # scripts/update.sh from this checkout. --force skips the version
+ # gate so the full deploy → migrate → switch → health path always
+ # executes.
+ sudo env PATH="$PATH" SERVERKIT_NO_SELF_UPDATE=1 bash scripts/update.sh --force
+
+ - name: Panel healthy after update
+ run: |
+ for i in $(seq 1 45); do
+ curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
+ echo "backend healthy after update (probe $i)"; exit 0; }
+ sleep 2
+ done
+ echo "backend never became healthy after update" >&2
+ exit 1
+
+ - name: Version gate skips a redundant re-run
+ run: |
+ # A second run without --force should hit the version gate and skip.
+ # The message check is advisory (the remote may have advanced between
+ # the two runs, making a second full update legitimate) — only the
+ # exit code gates.
+ out="$(sudo env PATH="$PATH" SERVERKIT_NO_SELF_UPDATE=1 bash scripts/update.sh)"
+ printf '%s\n' "$out"
+ printf '%s' "$out" | grep -qi "already up to date" \
+ || echo "::warning::second run did not report 'Already up to date' (remote may have advanced)"
+
+ - name: Dump install/update state
+ if: failure()
+ run: |
+ set +e
+ echo '===== /var/log/serverkit (updater logs) ====='
+ sudo sh -c 'for f in /var/log/serverkit/*.log; do
+ [ -f "$f" ] || continue
+ echo "----- $f (last 300 lines) -----"
+ tail -n 300 "$f"
+ done'
+ echo '===== systemd ====='
+ sudo systemctl status serverkit --no-pager -l
+ sudo journalctl -u serverkit --no-pager -n 200
+ echo '===== nginx ====='
+ sudo systemctl status nginx --no-pager -l
+ sudo nginx -t
+ sudo tail -n 100 /var/log/nginx/error.log
+ echo '===== layout ====='
+ ls -la /opt/ /opt/serverkit/
+ true
+
+ shellcheck-advisory:
+ name: shellcheck full ruleset (advisory)
+ # Everything shellcheck can say about the two scripts users actually pipe
+ # to bash, at maximum sensitivity. Deliberately non-gating: --enable=all
+ # carries style opinions the codebase does not follow, so this exists to be
+ # read when touching install.sh / scripts/update.sh — not chased to green.
+ # The gating checks stay in the lint + unit job above.
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: ShellCheck --enable=all (advisory — never fails the build)
+ continue-on-error: true
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -y -qq shellcheck
+ shellcheck --enable=all install.sh scripts/update.sh
diff --git a/.github/workflows/scripts-nightly.yml b/.github/workflows/scripts-nightly.yml
new file mode 100644
index 00000000..12168750
--- /dev/null
+++ b/.github/workflows/scripts-nightly.yml
@@ -0,0 +1,184 @@
+name: Scripts Nightly E2E
+
+# Nightly, non-blocking end-to-end coverage for the artifacts users actually
+# receive — deeper (and slower) than the PR-time Scripts CI:
+#
+# tarball-install builds the release tarball with scripts/build-release.sh
+# and INSTALLS from it through the installer's offline
+# path. The PR-time release smoke only proves the tarball
+# *builds*; this proves the shipped tree lands and the
+# pre-built, relocated venv actually runs at
+# /opt/serverkit/venv (the shipped-venv portability class
+# of breakage).
+#
+# update-from-release installs the latest PUBLISHED release, then runs THIS
+# checkout's updater over it — a genuine cross-version
+# update, which catches carry-forward/migration
+# regressions that a same-version forced update cannot.
+# The published baseline may itself be old or broken;
+# that is tolerated loudly — only a failure in this
+# checkout's updater fails the job.
+#
+# Both jobs mutate their runner (systemd units, nginx, /opt/serverkit), so each
+# runs alone on a fresh VM. There is deliberately no pull_request trigger:
+# nightly results must never block a PR.
+
+on:
+ schedule:
+ # 05:17 UTC — off the top-of-hour spike, after the day's merges settle.
+ - cron: '17 5 * * *'
+ workflow_dispatch:
+
+jobs:
+ tarball-install:
+ name: Install from a freshly built release tarball
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+
+ - name: Install build dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y python3-venv python3-pip rsync
+
+ - name: Build the release tarball
+ run: bash scripts/build-release.sh
+
+ - name: Install from the local tarball (offline release path)
+ run: |
+ # SERVERKIT_OFFLINE_TARBALL is the installer's documented local-tarball
+ # mechanism: fetch_release uses it verbatim instead of downloading.
+ # `sudo env`, not `... | sudo bash`, so the env contract survives
+ # sudo's environment reset; PATH is forwarded so the runner's
+ # toolchain stays visible to root.
+ TARBALL="$GITHUB_WORKSPACE/serverkit-v$(tr -d '\n\r ' < VERSION)-linux-amd64.tar.gz"
+ ls -lh "$TARBALL"
+ sudo env PATH="$PATH" \
+ INSTALL_FROM_RELEASE=1 \
+ SERVERKIT_OFFLINE_TARBALL="$TARBALL" \
+ SERVERKIT_SKIP_SSL=1 \
+ bash install.sh
+
+ - name: Assert the tree landed and the shipped venv works
+ run: |
+ readlink -f /opt/serverkit
+ test -d /opt/serverkit/backend/app
+ test -d /opt/serverkit/frontend/dist
+ test -x /opt/serverkit/serverkit
+ [ "$(tr -d '\n\r ' < VERSION)" = "$(tr -d '\n\r ' < /opt/serverkit/VERSION)" ]
+ test -x /opt/serverkit/venv/bin/python
+ # The venv was built elsewhere (the build step) and relocated by
+ # build-release.sh — importing the app's core deps from its installed
+ # path is the actual portability proof.
+ sudo /opt/serverkit/venv/bin/python -c 'import flask, sqlalchemy, gunicorn; print("shipped venv imports OK")'
+
+ - name: Panel healthy from the tarball install
+ run: |
+ for i in $(seq 1 30); do
+ curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
+ echo "backend healthy (probe $i)"; exit 0; }
+ sleep 2
+ done
+ echo "backend never became healthy after the tarball install" >&2
+ exit 1
+
+ - name: Dump install state
+ if: failure()
+ run: |
+ set +e
+ echo '===== /var/log/serverkit ====='
+ sudo sh -c 'for f in /var/log/serverkit/*.log; do
+ [ -f "$f" ] || continue
+ echo "----- $f (last 300 lines) -----"
+ tail -n 300 "$f"
+ done'
+ echo '===== systemd ====='
+ sudo systemctl status serverkit --no-pager -l
+ sudo journalctl -u serverkit --no-pager -n 200
+ echo '===== nginx ====='
+ sudo systemctl status nginx --no-pager -l
+ sudo nginx -t
+ sudo tail -n 100 /var/log/nginx/error.log
+ echo '===== layout ====='
+ ls -la /opt/ /opt/serverkit/
+ true
+
+ update-from-release:
+ name: Update from the latest published release
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Install the latest published release (baseline)
+ id: baseline
+ run: |
+ # INSTALL_FROM_RELEASE=1 with no version pin resolves and installs the
+ # newest published tarball — the state a real, older box is in. The
+ # baseline is not this checkout's product: if it fails to install or
+ # come up healthy, warn loudly and skip the update leg instead of
+ # failing — this job only gates regressions in THIS checkout's updater.
+ ok=true
+ sudo env PATH="$PATH" INSTALL_FROM_RELEASE=1 SERVERKIT_SKIP_SSL=1 \
+ bash install.sh || ok=false
+ if [ "$ok" = "true" ]; then
+ ok=false
+ for i in $(seq 1 30); do
+ curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && { ok=true; break; }
+ sleep 2
+ done
+ fi
+ echo "baseline version: $(tr -d '\n\r ' < /opt/serverkit/VERSION 2>/dev/null || echo none)"
+ echo "ok=$ok" >> "$GITHUB_OUTPUT"
+ [ "$ok" = "true" ] \
+ || echo "::warning::baseline install of the published release failed or never got healthy — skipping the update leg (not a regression in this checkout)"
+
+ - name: Run THIS checkout's updater over the published install
+ if: steps.baseline.outputs.ok == 'true'
+ run: |
+ # SERVERKIT_NO_SELF_UPDATE=1: do not re-exec into the published
+ # updater — the object under test is scripts/update.sh from this
+ # checkout. --force: never let the version gate skip the run.
+ sudo env PATH="$PATH" SERVERKIT_NO_SELF_UPDATE=1 bash scripts/update.sh --force
+
+ - name: Panel healthy after the cross-version update
+ if: steps.baseline.outputs.ok == 'true'
+ run: |
+ for i in $(seq 1 45); do
+ curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
+ echo "backend healthy after update to version $(tr -d '\n\r ' < /opt/serverkit/VERSION 2>/dev/null || echo unknown) (probe $i)"
+ exit 0
+ }
+ sleep 2
+ done
+ echo "backend never became healthy after the cross-version update" >&2
+ exit 1
+
+ - name: Dump install/update state
+ if: failure()
+ run: |
+ set +e
+ echo '===== /var/log/serverkit (updater logs) ====='
+ sudo sh -c 'for f in /var/log/serverkit/*.log; do
+ [ -f "$f" ] || continue
+ echo "----- $f (last 300 lines) -----"
+ tail -n 300 "$f"
+ done'
+ echo '===== systemd ====='
+ sudo systemctl status serverkit --no-pager -l
+ sudo journalctl -u serverkit --no-pager -n 200
+ echo '===== nginx ====='
+ sudo systemctl status nginx --no-pager -l
+ sudo nginx -t
+ sudo tail -n 100 /var/log/nginx/error.log
+ echo '===== layout ====='
+ ls -la /opt/ /opt/serverkit/
+ true