Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Agents controllers update themselves to the release Core names (#811)
Louis Amaudruzecd8b64be541
diff --git a/.env.example b/.env.exampleindex 556e78af0..800566a1e 100644--- a/.env.example+++ b/.env.example@@ -156,6 +156,12 @@ AGENT_REGISTRATION_TOKEN=# `agent_management`. When on, CONTROLLER_TOKEN_SECRET is required: at least# 32 random characters, separate from SECRET_KEYS, e.g. `openssl rand -base64 48`.CONTROLLER_TOKEN_SECRET=+# The agents controller release Core tells standalone and cloud controllers to+# run: they install it when idle, checking this SHA-256. All three or none;+# none leaves every controller on the version it has.+# CONTROLLER_RELEASE_VERSION=+# CONTROLLER_RELEASE_URL=+# CONTROLLER_RELEASE_SHA256=# ── Feature flags ────────────────────────────────────────────────────────────# Flags turned on for this deployment, comma-separated (e.g.diff --git a/.github/workflows/switch-dev-cleanup.yml b/.github/workflows/switch-dev-cleanup.ymlindex 5a276d1a4..d114990e7 100644--- a/.github/workflows/switch-dev-cleanup.yml+++ b/.github/workflows/switch-dev-cleanup.yml@@ -1,9 +1,10 @@name: switch dev cleanup# Prunes old dev builds — the images and chart every push to main publishes-# under ghcr.io/<owner>/dev/ (see switch-release.yml) — so the dev packages do-# not grow without bound. Only the dev/ packages are touched; releases and RCs-# live in packages of their own. A build is deleted once it is older than 30+# under ghcr.io/<owner>/dev/, and the agents controller it publishes as a+# GitHub prerelease (see switch-release.yml) — so they do not grow without+# bound. Only the dev/ packages and the dev controller prereleases are+# touched; releases and RCs live apart. A build is deleted once it is older than 30# days and outside the 20 newest, so the dev environment's pin always has# something recent to point at. See scripts/prune_dev_packages.py.@@ -18,7 +19,8 @@ on:default: truepermissions:- contents: read+ # To delete the dev controller prereleases and their tags.+ contents: writepackages: writeconcurrency:@@ -38,6 +40,6 @@ jobs:GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}run: |- args=(--owner "${{ github.repository_owner }}" --max-age-days 30 --keep 20)+ args=(--owner "${{ github.repository_owner }}" --repository "${{ github.repository }}" --max-age-days 30 --keep 20)[ "$DRY_RUN" != true ] || args+=(--dry-run)python3 scripts/prune_dev_packages.py "${args[@]}"diff --git a/.github/workflows/switch-release.yml b/.github/workflows/switch-release.ymlindex 1c87968af..31fef177d 100644--- a/.github/workflows/switch-release.yml+++ b/.github/workflows/switch-release.yml@@ -41,6 +41,13 @@ name: switch release# whose charts were partly pushed. Dev chart builds also move a `main` tag# that development follows.#+# Every release, release candidate and dev build also publishes the agents+# controller at its version as the `switch-agent-controller-v<version>` GitHub+# release (a prerelease for an RC or a dev build; switch-dev-cleanup.yml prunes+# only the dev ones), and pins+# that package into the switch chart's switchCore.controllerRelease, so Core+# tells controllers to run the controller of the Switch they talk to.+## Artifacts go to GHCR by default, and the packages are public alongside the# repo. REGISTRY / IMAGE_NAMESPACE are the only things to change to retarget# another registry (e.g. ECR) — a config flip, not a rewrite.@@ -299,13 +306,162 @@ jobs:if-no-files-found: errorretention-days: 7+ # The agents controller (console/packages/agent-controller) at this release's+ # version, published as the `switch-agent-controller-v<version>` GitHub+ # release that `install.sh` and `switch-agent-controller update` already read.+ # The chart below is pinned to it (switchCore.controllerRelease), so Core+ # tells every controller to run the controller of the Switch it talks to.+ # A release candidate and a dev build of main publish theirs too, as a GitHub+ # prerelease (which `install.sh` and `update` leave out), so the controllers+ # of a deployment running an RC, or the dev environment, follow it. A dry run+ # builds, tests and packs the controller but publishes nothing.+ controller:+ name: Package & publish agents controller+ needs: prep+ runs-on: ubuntu-latest+ permissions:+ contents: write+ outputs:+ version: ${{ steps.version.outputs.version }}+ url: ${{ steps.version.outputs.url }}+ sha256: ${{ steps.sha256.outputs.sha256 }}+ defaults:+ run:+ working-directory: console+ steps:+ - name: Resolve version+ id: version+ working-directory: .+ env:+ VERSION: ${{ needs.prep.outputs.version }}+ run: |+ tag="switch-agent-controller-v$VERSION"+ {+ echo "version=$VERSION"+ echo "tag=$tag"+ echo "url=https://github.com/$GITHUB_REPOSITORY/releases/download/$tag/switch-agent-controller-$VERSION.tgz"+ } | tee -a "$GITHUB_OUTPUT"++ - name: Checkout+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0++ - name: Set up pnpm+ uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4+ with:+ version: 11.5.3++ - name: Set up Node+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version-file: console/.nvmrc+ cache: pnpm+ cache-dependency-path: console/pnpm-lock.yaml++ - name: Stamp the version+ # On the runner, not committed: the CLI reports the version it was built with.+ working-directory: console/packages/agent-controller+ env:+ VERSION: ${{ steps.version.outputs.version }}+ run: npm pkg set version="$VERSION"++ - name: Install dependencies+ run: pnpm install --frozen-lockfile++ - name: Build the workspace packages+ run: pnpm -r --filter './packages/**' run build++ - name: Test the controller+ run: pnpm --filter @switch-console/agent-controller run test++ - name: Package+ run: pnpm --filter @switch-console/agent-controller run package++ - name: Smoke-test the package+ env:+ VERSION: ${{ steps.version.outputs.version }}+ run: |+ prefix="$(mktemp -d)"+ npm install --global --prefix "$prefix" "packages/agent-controller/dist-package/switch-agent-controller-$VERSION.tgz"+ test "$("$prefix/bin/switch-agent-controller" --version)" = "$VERSION"+ "$prefix/bin/switch-agent-controller" --help > /dev/null++ - name: Hash the package+ id: sha256+ env:+ VERSION: ${{ steps.version.outputs.version }}+ run: |+ sha256="$(sha256sum "packages/agent-controller/dist-package/switch-agent-controller-$VERSION.tgz" | cut -d' ' -f1)"+ echo "sha256=$sha256" | tee -a "$GITHUB_OUTPUT"++ - name: Release+ # The tag is created from this commit by the release itself. A tag+ # pushed with the workflow token starts no workflow, so+ # agent-controller-release.yml does not publish it a second time.+ if: needs.prep.outputs.push == 'true'+ env:+ GH_TOKEN: ${{ github.token }}+ VERSION: ${{ steps.version.outputs.version }}+ TAG: ${{ steps.version.outputs.tag }}+ SHA256: ${{ steps.sha256.outputs.sha256 }}+ CHANNEL: ${{ needs.prep.outputs.channel }}+ run: |+ package="packages/agent-controller/dist-package/switch-agent-controller-$VERSION.tgz"+ if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then+ # A re-run of this release: the package already published must be+ # the one just built, or the chart would pin a hash nothing serves.+ existing="$(mktemp -d)"+ gh release download "$TAG" --repo "$GITHUB_REPOSITORY" \+ --pattern "switch-agent-controller-$VERSION.tgz" --dir "$existing"+ published="$(sha256sum "$existing/switch-agent-controller-$VERSION.tgz" | cut -d' ' -f1)"+ if [ "$published" != "$SHA256" ]; then+ echo "::error::$TAG is already released with a different package (sha256 $published, this build $SHA256). Delete that release and its tag, then re-run."+ exit 1+ fi+ echo "$TAG is already released with this package."+ exit 0+ fi+ if [ "$CHANNEL" = rc ]; then+ gh release create "$TAG" "$package" \+ --repo "$GITHUB_REPOSITORY" \+ --target "$GITHUB_SHA" \+ --title "switch-agent-controller $VERSION (release candidate)" \+ --notes "The Switch agents controller built for the release candidate Switch $VERSION, at $GITHUB_SHA. Controllers connected to a Switch running that candidate update to it on their own once idle; bake it into a machine image for that candidate. \`install.sh\` and \`switch-agent-controller update\` leave it out.++ SHA-256: \`$SHA256\`" \+ --prerelease \+ --latest=false+ exit 0+ fi+ if [ "$CHANNEL" = dev ]; then+ gh release create "$TAG" "$package" \+ --repo "$GITHUB_REPOSITORY" \+ --target "$GITHUB_SHA" \+ --title "switch-agent-controller $VERSION (dev build of main)" \+ --notes "The Switch agents controller built from main at $GITHUB_SHA, for the dev environment, whose controllers update to it on their own. Not for installing by hand; switch-dev-cleanup prunes it.++ SHA-256: \`$SHA256\`" \+ --prerelease \+ --latest=false+ exit 0+ fi+ gh release create "$TAG" "$package" packages/agent-controller/install.sh \+ --repo "$GITHUB_REPOSITORY" \+ --target "$GITHUB_SHA" \+ --title "switch-agent-controller $VERSION" \+ --notes "The Switch agents controller $VERSION, released with Switch $VERSION, for running managed agents on your own Linux or macOS machines.++ SHA-256: \`$SHA256\`++ Install (Node 22.13 or later): \`npm install --global https://github.com/$GITHUB_REPOSITORY/releases/download/$TAG/switch-agent-controller-$VERSION.tgz\`, or run \`install.sh\`. Then enroll with a code from the Machines page and run \`switch-agent-controller install-service\`. A controller connected to Switch $VERSION updates to this release on its own once it is idle." \+ --latest=false+chart:name: Package & publish Helm charts# After the images: each chart carries its images' digests, so it cannot be# packaged before they exist, and a chart is never published pointing at an# image that failed to push. The charts come from# scripts/pin_chart_images.py (`charts`), like the image matrix.- needs: [prep, images]+ needs: [prep, images, controller]runs-on: ubuntu-latestpermissions:contents: read@@ -347,6 +503,38 @@ jobs:docker buildx imagetools inspect "$REGISTRY/$NAMESPACE/$(basename "$file")@$(cat "$file")" >/dev/nulldone+ - name: Pin the controller release+ # Written into the switch chart's default values on the runner, never+ # committed, before the chart is pinned and packaged.+ if: ${{ needs.controller.outputs.version != '' }}+ env:+ CONTROLLER_VERSION: ${{ needs.controller.outputs.version }}+ CONTROLLER_URL: ${{ needs.controller.outputs.url }}+ CONTROLLER_SHA256: ${{ needs.controller.outputs.sha256 }}+ run: |+ python3 - <<'PY'+ import os, re+ path = "deploy/remote/helm/switch/values.yaml"+ text = open(path).read()+ block = re.search(r"(?m)^ controllerRelease:\n(?:(?: .*)?\n)+", text)+ if block is None:+ raise SystemExit(f"no switchCore.controllerRelease block in {path}")+ pinned = block.group(0)+ for key, env in (("version", "CONTROLLER_VERSION"), ("url", "CONTROLLER_URL"), ("sha256", "CONTROLLER_SHA256")):+ value = os.environ[env]+ if not value:+ raise SystemExit(f"{env} is empty")+ pinned, n = re.subn(rf'(?m)^ {key}: ""$', lambda _: f' {key}: "{value}"', pinned, count=1)+ if n != 1:+ raise SystemExit(f'no empty `{key}: ""` under switchCore.controllerRelease in {path}')+ open(path, "w").write(text[: block.start()] + pinned + text[block.end() :])+ PY+ sed -n '/^ controllerRelease:/,/^ [a-zA-Z]/p' deploy/remote/helm/switch/values.yaml+ # The check is only that switch-core gets the release.+ helm template switch deploy/remote/helm/switch \+ -f deploy/remote/helm/render-check-values.yaml \+ | grep -A1 -E 'name: CONTROLLER_RELEASE_(VERSION|URL|SHA256)$'+- name: Pin, lint, package and check each chart# A dry run pushed no images, so it pins placeholder digests: the pin# and the checks still run, and nothing is published. The checks lintdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex f770b3eee..5496d85ce 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -75,6 +75,22 @@ version of their own to them without also giving them a release of their own.`secrets.gatewaySmtpUsername`, or from `GATEWAY_SMTP_USERNAME` in`secrets.existingSecret`, instead of `switchCore.smtp.username`. Settingboth is refused at render time.+- **Agents controllers update to the release Core names.** With+ `CONTROLLER_RELEASE_VERSION`, `CONTROLLER_RELEASE_URL` and+ `CONTROLLER_RELEASE_SHA256` (Helm `switchCore.controllerRelease`; all three or+ none, refused at startup and at render otherwise), the status answer carries+ `controller_release: {version, url, sha256}` (null for a Switch Console+ controller, which updates with Console), and a controller on another+ version fetches that package, checks its SHA-256 and installs it once idle.+ `GET /gateway/management/controllers` shows each controller's `release`+ (`{wanted, running, current}`, null when none is named), and its status keeps+ the `update` the controller reports. A Switch release now also+ publishes the controller at its version as `switch-agent-controller-v<version>`,+ and its chart names that package; a release candidate and a dev build of main+ do the same as a prerelease, so the controllers of a deployment running an RC,+ and the dev environment's, follow it. A controller+ run as a service installs a release beside the one it runs and keeps the one+ before. Roll back by pointing the values at the previous release.#### Security- **An agent icon URL must spell its host plainly.** A link with a backslash,diff --git a/RELEASING.md b/RELEASING.mdindex f8528438d..1f205968f 100644--- a/RELEASING.md+++ b/RELEASING.md@@ -77,10 +77,15 @@ peer still on it, and it can never be raised past what is running in the field.- builds multi-arch (`linux/amd64`, `linux/arm64`) images for `switch-core`,`gateway`, and `setup` and pushes them to the container registry tagged`<version>` and `latest`;+ - builds, tests and packs the agents controller at the same version and+ publishes it as the `switch-agent-controller-v<version>` GitHub release+ (see "The controller a Switch release names" below);- once the images are pushed, writes each image's digest into the chart's- `values.yaml`, packages the chart at the same version, checks the packaged- chart renders every first-party image by digest, and pushes it as an OCI- artifact to the same registry (see [Pinned by digest](#pinned-by-digest)).+ `values.yaml` and that controller package's version, URL and SHA-256 into+ `switchCore.controllerRelease`, packages the chart at the same version,+ checks the packaged chart renders every first-party image by digest, and+ pushes it as an OCI artifact to the same registry (see+ [Pinned by digest](#pinned-by-digest)).`workflow_dispatch` runs the build without pushing, for verification. Itpins placeholder digests, so the pin and the check run too.@@ -103,6 +108,12 @@ git push origin switch-v0.30.0-rc.1`core/pyproject.toml` and not already released as `switch-v0.30.0`.- **Never `latest`.** Images, chart and compose are pushed under `0.30.0-rc.1` only.`helm install` skips the chart unless given `--devel` or that exact `--version`.+- **Its own controller, as a prerelease.** An RC publishes the controller as+ `switch-agent-controller-v0.30.0-rc.1`, a GitHub prerelease that `install.sh` and+ `switch-agent-controller update` leave out, and its chart names it in+ `switchCore.controllerRelease`: controllers talking to an RC update to it once+ idle, as they would for a release. `0.30.0-rc.1` sorts before `0.30.0`, so the+ release that follows moves them on. The dev cleanup never prunes it.- **The server reports the RC.** The workflow stamps `0.30.0rc1` (the PEP 440 form)into `core/pyproject.toml` on the runner, so the running server's version sayswhich candidate it is. Nothing is committed.@@ -129,10 +140,17 @@ building from source at deploy time.before the next release or RC. The server reports `0.29.1.dev512` (PEP 440),stamped on the runner like an RC. A semver pre-release, so `helm install`skips it unless given that exact `--version`. No compose artifact.+- **A dev controller too.** The run publishes the agents controller at the dev+ version as a GitHub *prerelease*, `switch-agent-controller-v<dev version>`+ (package only, no `install.sh`), and pins it into the dev switch chart's+ `switchCore.controllerRelease`, so the dev environment's controllers follow+ main the way a release's follow that release. `install.sh` and+ `switch-agent-controller update` leave prereleases out.- **The newest push wins.** Pushes to main share a concurrency group, so a devbuild still running when the next merge lands is cancelled.- **Pruned.** `switch-dev-cleanup.yml` runs daily and deletes dev builds older- than 30 days that are not among the 20 newest (`scripts/prune_dev_packages.py`;+ than 30 days that are not among the 20 newest, the dev controller+ prereleases and their tags included (`scripts/prune_dev_packages.py`;dispatch it with `dry_run` to see what it would delete).- **Not gated on PR CI.** It builds whatever reached main; what an environmentruns is decided by its pin, not by the build existing.@@ -280,6 +298,39 @@ git push origin switch-agent-controller-v0.2.0`workflow_dispatch` runs everything except the release, for verification.+### The controller a Switch release names++A Switch release (`switch-v<version>`) also publishes the controller+(an RC and a dev build of main do too, as a prerelease: see+[Release candidates](#release-candidates) and [Dev builds](#dev-builds)),+built from the same commit, as `switch-agent-controller-v<version>`: the same+GitHub release this workflow makes (package and `install.sh`, never Latest), so+`install.sh` and `switch-agent-controller update` find it as they find any other.+Its tag is created by the release with the workflow token, which starts no+workflow, so this workflow does not publish it again. If that release already+exists (a re-run), the switch-release workflow checks that its package is the+one it just built and fails otherwise.++The chart from that release names the package in `switchCore.controllerRelease`+(`version`, `url`, `sha256`), which switch-core gets as+`CONTROLLER_RELEASE_VERSION`, `_URL` and `_SHA256` and hands every controller in+its status answer (except Switch Console's own, which updates with Console). A+controller running another version downloads the package,+refuses it unless its SHA-256 matches, and installs it and restarts once none of+its agents has a session running. The owner's controller list+(`GET /gateway/management/controllers`, `release.current`) shows which are not yet+on it.++- **Rollback:** point the three values at the previous release's package (its+ version, its `.../switch-agent-controller-v<old>/switch-agent-controller-<old>.tgz`+ URL, and the SHA-256 in its release notes) and upgrade the chart. Setting none of+ them stops telling controllers anything; they keep the version they have.+- **Versions now follow Switch.** Controllers released this way carry the Switch+ version, which is well above the controller's own earlier `0.1.x` line. A+ controller release tagged by hand still works, but take its version from the+ Switch version it ships beside, or it is either overtaken by the next Switch+ release or ahead of the one Core names.+## Where artifacts are publishedThe images, the chart, and the standalone compose artifact all go to **GitHubdiff --git a/console/packages/agent-controller/README.md b/console/packages/agent-controller/README.mdindex 137b57454..15ae74733 100644--- a/console/packages/agent-controller/README.md+++ b/console/packages/agent-controller/README.md@@ -119,6 +119,7 @@ switch-agent-controller update [--check]| `4` | Another instance of this controller, with the same identity, opened the controller stream after this one, which took it over. | No || `5` | The server does not speak this controller's protocol (`protocol_unsupported`). | No: update the controller || `6` | The server knows no controller by this credential (`invalid_credential`): it was replaced, or its key was deleted. | No: enroll again |+ | `7` | It installed the release Switch names and is restarting into it (see [Updates](#updates)). | Yes, at once |Code `2` covers: an unknown command, option or missing argument; a serverURL that is not one, or is plain `http` to a host that is not loopback; a@@ -158,7 +159,10 @@ Linux it also starts at boot and survives logout only with lingering onkeeps the `PATH` of the shell that installed it, so the provider CLIs found then arefound by the service. A data directory other than the default gets a service namewith a hash of its path, so two controllers on one account each get their own.-`uninstall-service` stops and removes it and keeps the enrollment.+The service runs the controller through `<data dir>/releases/current`, which+`install-service` points at the npm install it was run from (see+[Updates](#updates)). `uninstall-service` stops and removes it, and the releases it+installed, and keeps the enrollment.On Linux without a systemd user manager (some containers and minimal VMs),`install-service` refuses. Run `run` under your own supervisor instead, with the@@ -227,12 +231,25 @@ user's default one):`switch-agent-controller-<uid>.service`, run with`--agent-runtime separate-user` and the agents' group as a supplementary group,so it can read what its agents write. It replaces the user service, which must- be uninstalled first.+ be uninstalled first;+- a releases directory, `/var/lib/switch-agent-controller-<uid>/releases`+ (`--releases-dir`), owned by the controller's user and readable by everyone,+ where the controller installs the releases Switch names (see+ [Updates](#updates)). The controller's unit runs+ `<releases>/current/lib/node_modules/switch-agent-controller/cli.mjs` and the+ agents' units run the shared-host bundle beside it, so both follow `current`.+ The setup points `current` at the release it runs from (for example+ `/usr/lib` after `sudo npm install --global`) only when `current` is missing,+ names no readable release, or names an older one: running it again with the+ same or an older release keeps an update applied since. The directory must not+ be under a home, the data directory or the agents' directories, which agents+ cannot reach. A controller run from a checkout gets none.The setup is written to `/etc/switch-agent-controller/separate-users-<uid>.json`.Running the setup again rewrites everything and adds agent users. `sudoswitch-agent-controller uninstall-service --separate-users` stops the controller-and its agents, then removes what the setup made except the agents' directories.+and its agents, then removes what the setup made except the agents' directories+and the releases directory.Each agent claims a free user from the pool when it is first started, and runs as`switch-agent-<uid>@<NN>.service`. systemd restarts an agent host that crashes. The@@ -324,9 +341,57 @@ directory remembers it:### Updates-`update` installs the newest release with npm and restarts the service if it runs;-`update --check` only says whether there is one. `run` logs a warning when a newer-release exists, and `doctor` shows it.+When Switch names a release for this controller, `run` follows it; otherwise+`update` does it by hand.++**The release Switch names.** A status response can carry+`controller_release: {"version", "url", "sha256"}`. When it names a version other+than the one running (newer or older: Switch can roll back), the controller:++1. waits until no session runs on the machine (`sessions_running` is 0), checking+ at each status report, and reports `update: {"target", "state": "waiting_idle"}`+ in its status meanwhile;+2. downloads the package into `<data dir>/updates/` (at most 200 MB) and checks+ its SHA-256 against the one Switch gave, the only one trusted; a package that+ differs is deleted;+3. installs it from that file, never from the URL, checks that the installed CLI+ says `--version` is that version, checks again that no session started+ meanwhile, and switches to it;+4. exits with code `7`, and its service starts it again on the new release.++A failed attempt is reported (`update: {"target", "state": "failed", "problem"}`)+and logged, and the same release is not tried again for an hour; the last+attempt is kept in the data directory, so a restart does not retry it either.+Absent or null, `controller_release` leaves the controller alone; one that cannot+be read is logged and treated as absent. Switch Console's own controller (run+with `--credential-stdin`) never updates itself: it updates with Console.++Where it installs depends on how the controller was installed:++- **As a service, of your user or with `--separate-users`:** into its releases+ directory, `<data dir>/releases` for a service of your user (`--releases-dir`,+ which `install-service` passes to `run`) or the one the separate-users setup+ names (see [Each agent as a Linux user of its own](#each-agent-as-a-linux-user-of-its-own)).+ Each release is `<releases>/<version>/`, an npm prefix: it installs into+ `<version>.partial`, checks it, renames it, then points `<releases>/current`, which+ the service runs through, at it in one step. It keeps the release it replaced and+ at most one older one, and never touches a prefix outside the directory, such as+ the npm install `current` first named. A failed install leaves `current` as it+ was. A service of your user installed before releases directories updates in+ place, as below, until `install-service` is run again.+- **A plain npm install, not run as a service:** with `npm install --global+ --prefix <prefix>`, over the prefix the running controller came from, as `update`+ does. A controller run from a checkout does not update itself, and reports why,+ as does one on Windows. Under systemd (which sets `INVOCATION_ID`) or launchd+ (`--launchd`) it exits `7` to be restarted; run any other way, it installs,+ reports `restart_needed` and logs that it must be restarted, and keeps running.++**By hand.** `update` installs the newest published release with npm and restarts+the service if it runs; `update --check` only says whether there is one. `run` logs+a warning when a newer release exists, and `doctor` shows it. `update` refuses to+install into a releases directory, which only its controller installs into. Run from+the npm install, it installs there and points the service's `current` back at it when+that is newer than what `current` names.### Checking a machine@@ -343,8 +408,10 @@ To run it under another supervisor, have it run `run` and restart it on exit codconfiguration is fixed, nor on `3`, because a revoked controller must be enrolledagain, nor on `4`, because two instances would take the stream from each other inturn, nor on `5` or `6`, because the server has said it will refuse this controller-the same way again. With systemd, `Restart=on-failure` and-`RestartPreventExitStatus=2 3 4 5 6`.+the same way again. Do restart it on `7`: it installed the release Switch names and+ends to run it. With systemd, `Restart=on-failure` and+`RestartPreventExitStatus=2 3 4 5 6`. A controller run outside systemd and launchd+installs the release Switch names without exiting; restart it to run it.## Run by a parent process@@ -542,6 +609,8 @@ It holds:| `secrets/controller-credential` | The controller credential, with the `file` secret store (see below). Absent with a keychain store, or when the credential is handed over with `--credential-stdin`. || `agents/<id>/credentials.json` | Each agent's relay endpoint, relay token and hub, in the layout the shared host reads. No Switch credential. || `agent hosts/<id>/` | Each agent's agent host state root: `watch.json`, `config.json`, `health.json` (what `status` reads), its journal, and `supervisor/failure.json` once it has failed for good. |+| `updates/` | A release being downloaded to update to (deleted once installed or refused), and npm's cache. The last update attempt is kept in `controller.db`. |+| `releases/` | With `install-service`: the releases the controller installed for itself, each an npm prefix, and `current`, the link the service runs through. |An agent whose definition names no directory works in`~/.switch/agents/<server>/<name>/`, where `<server>` is the server's addressdiff --git a/console/packages/agent-controller/scripts/package.mjs b/console/packages/agent-controller/scripts/package.mjsindex 2c6079e22..a607fd812 100644--- a/console/packages/agent-controller/scripts/package.mjs+++ b/console/packages/agent-controller/scripts/package.mjs@@ -19,8 +19,12 @@ const out = join(root, 'dist-package');const stage = join(out, 'package');const manifest = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));const { version } = manifest;-if (!/^\d+\.\d+\.\d+$/.test(version))- throw new Error(`The package version must be x.y.z to be released, not '${version}'.`);+// The versions the controller follows (RELEASE_VERSION in src/update.ts): a+// release's x.y.z, or a dev build's x.y.z-<prerelease>.+if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*)?$/.test(version))+ throw new Error(+ `The package version must be x.y.z or x.y.z-<prerelease> to be released, not '${version}'.`+ );/** Kept out, as in Switch Console's own bundle: the controller must stay headless. */const FORBIDDEN = ['electron', 'better-sqlite3', 'drizzle-orm'];diff --git a/console/packages/agent-controller/src/controller.test.ts b/console/packages/agent-controller/src/controller.test.tsindex f4ca96472..cff123b23 100644--- a/console/packages/agent-controller/src/controller.test.ts+++ b/console/packages/agent-controller/src/controller.test.ts@@ -19,6 +19,7 @@ import { silentLogger } from './log';import type { LocalRelay } from './relay';import type { AgentAssignment, StatusReport } from './schemas';import { CONTROLLER_CREDENTIAL, FileSecretStore, MemorySecretStore } from './secrets';+import { type ReleaseInstaller, SelfUpdater, type UpdateAttempt } from './self-update';import { ControllerStore } from './store';import { buildWatcherTemplate } from './template';import { FakeCore } from './testing/fake-core';@@ -82,6 +83,7 @@ function deps(server = core.url): ControllerDeps {dataDir: dir,workspacesFor: () => join(dir, 'workspaces'),version: '0.1.0',+ selfUpdate: null,now: Date.now,random: () => 0,timing: {@@ -633,6 +635,71 @@ describe('runController', () => {});});+describe('the release Switch names', () => {+ function selfUpdate() {+ const installed: string[] = [];+ const installer: ReleaseInstaller = {+ unavailable: null,+ prepare: async () => {},+ activate: async (version) => void installed.push(version),+ };+ let attempt: UpdateAttempt | null = null;+ const updater = new SelfUpdater({+ version: '0.1.0',+ installer,+ download: async () => join(dir, 'never-written.tgz'),+ attempts: { last: () => attempt, save: (saved) => void (attempt = saved) },+ supervised: true,+ log: silentLogger,+ now: Date.now,+ });+ return { updater, installed };+ }++ const release = {+ version: '0.2.0',+ url: 'https://example.invalid/switch-agent-controller-0.2.0.tgz',+ sha256: 'c'.repeat(64),+ };++ it('is installed while no session runs, and the controller ends to restart on it', async () => {+ core.setAssignment({ revision: 1, agents: [agent(1)] });+ core.controllerRelease = release;+ const { updater, installed } = selfUpdate();+ running = runController({ ...deps(), selfUpdate: updater }, stop.signal);+ expect(await running).toBe('restarting');+ expect(installed).toEqual(['0.2.0']);+ });++ it('waits while a session runs, and says so in the status', async () => {+ core.setAssignment({ revision: 1, agents: [agent(1)] });+ const { updater, installed } = selfUpdate();+ running = runController({ ...deps(), selfUpdate: updater }, stop.signal);+ await waitFor(() => runtime.launches('agent-1').length === 1, 'the agent started');+ runtime.observation('agent-1').health!.placements = { 'session-1': 'room-a' };+ core.controllerRelease = release;+ await waitFor(+ () => core.statusReports.some((report) => report.update?.state === 'waiting_idle'),+ 'a status saying the update waits'+ );+ expect(core.statusReports.at(-1)?.update).toEqual({ target: '0.2.0', state: 'waiting_idle' });+ expect(installed).toEqual([]);+ runtime.observation('agent-1').health!.placements = {};+ expect(await running).toBe('restarting');+ expect(installed).toEqual(['0.2.0']);+ });++ it('is left alone by a controller that does not update itself', async () => {+ core.setAssignment({ revision: 1, agents: [agent(1)] });+ core.controllerRelease = release;+ running = runController(deps(), stop.signal);+ await waitFor(() => core.statusReports.length >= 2, 'two status reports');+ expect(core.statusReports.every((report) => report.update === undefined)).toBe(true);+ stop.abort();+ expect(await running).toBe('stopped');+ });+});+describe('ensureRelayCredentials', () => {const login = {status: 'connected' as const,diff --git a/console/packages/agent-controller/src/controller.ts b/console/packages/agent-controller/src/controller.tsindex aa953ff6c..78332a5a6 100644--- a/console/packages/agent-controller/src/controller.ts+++ b/console/packages/agent-controller/src/controller.ts@@ -28,6 +28,7 @@ import {} from './schemas';import { type GivenLogin, registerSealingKey, SealedLogins, sealingKeys } from './sealed-logins';import { CONTROLLER_CREDENTIAL, type SecretStore } from './secrets';+import type { SelfUpdater } from './self-update';import {type ProviderLocator,ProviderStatuses,@@ -89,6 +90,12 @@ export type ControllerDeps = {/** Where agents' working directories go when their definition names none, for a server. */workspacesFor: (server: string) => string;version: string;+ /**+ * Follows the release Switch names for this controller; null where the+ * controller is updated some other way (Switch Console's own, which+ * updates with Console).+ */+ selfUpdate: SelfUpdater | null;now: () => number;random: () => number;timing: ControllerTiming;@@ -103,13 +110,17 @@ export type ControllerDeps = {* controller's protocol, or knows no controller by its credential. Both are* refusals the server marks not retryable, and asking again cannot change* them, so this one ends rather than retrying forever.+ *+ * `restarting`: the release Switch names is installed, and the controller+ * ends for its service to start it again on that release.*/export type ControllerExit =| 'stopped'| 'revoked'| 'taken_over'| 'upgrade_required'- | 'credential_invalid';+ | 'credential_invalid'+ | 'restarting';/*** Makes an agent's credentials file name the relay, its hub, and a token the@@ -345,6 +356,12 @@ export async function runController(stop.abort();};+ let restarting = false;+ const restart = () => {+ restarting = true;+ stop.abort();+ };+const failed = (what: string, error: unknown) => {if (isRevoked(error)) {void revoke();@@ -482,13 +499,24 @@ export async function runController(});};+ const idle = async () => (await collector.collect(assignment)).machine.sessions_running === 0;reporter = new StatusReporter({- collect: () => collector.collect(assignment),+ collect: async () => {+ const collected = await collector.collect(assignment);+ const update = deps.selfUpdate?.status();+ return update ? { ...collected, update } : collected;+ },send: async (report) => {const answer = await client.putStatus(report);reportWithinS = answer.report_within_s;if (assignment === null || answer.assignment_revision > assignment.revision)void sync('status says the assignment moved on');+ if (deps.selfUpdate && !stop.signal.aborted)+ void deps.selfUpdate.offer(answer.controller_release, {+ idle,+ restart,+ signal: stop.signal,+ });},seq: () => store.nextStatusSeq(deps.now()),minGapMs: timing.statusMinGapMs,@@ -627,6 +655,8 @@ export async function runController(log.error('Switch knows no controller by this credential; it stops its agents and exits. Enroll this machine again.');+ if (restarting && !refused && !revocation)+ log.info('Exiting for the service to start this controller on its new release.');} finally {clearInterval(resync);clearInterval(poll);@@ -636,12 +666,14 @@ export async function runController(if (revocation) await revocation;await queue.drain();await reporter.idle();+ await deps.selfUpdate?.settled();} finally {await runtime.close();await relay.close();}}- return revocation ? 'revoked' : (refused ?? ending);+ if (revocation) return 'revoked';+ return refused ?? (restarting ? 'restarting' : ending);}/**diff --git a/console/packages/agent-controller/src/exit-codes.test.ts b/console/packages/agent-controller/src/exit-codes.test.tsindex 37263b896..579f7ef34 100644--- a/console/packages/agent-controller/src/exit-codes.test.ts+++ b/console/packages/agent-controller/src/exit-codes.test.ts@@ -6,6 +6,7 @@ import {EXIT_CONFIGURATION,EXIT_FAILURE,EXIT_OK,+ EXIT_RESTARTING,EXIT_REVOKED,EXIT_TAKEN_OVER,exitCodeFor,@@ -26,6 +27,7 @@ describe('exit codes', () => {expect([EXIT_OK, EXIT_FAILURE, EXIT_CONFIGURATION, EXIT_REVOKED, EXIT_TAKEN_OVER]).toEqual([0, 1, 2, 3, 4,]);+ expect(EXIT_RESTARTING).toBe(7);});});diff --git a/console/packages/agent-controller/src/exit-codes.ts b/console/packages/agent-controller/src/exit-codes.tsindex 7fcc77fdc..98b82bb54 100644--- a/console/packages/agent-controller/src/exit-codes.ts+++ b/console/packages/agent-controller/src/exit-codes.ts@@ -14,6 +14,12 @@ export const EXIT_TAKEN_OVER = 4;export const EXIT_UPGRADE_REQUIRED = 5;/** The server knows no controller by this credential. It has to be enrolled again. */export const EXIT_CREDENTIAL_INVALID = 6;+/**+ * The controller installed the release Switch names and ends to be started on+ * it: a service restarts it (systemd's `Restart=on-failure`, launchd's+ * `KeepAlive`), so 7 must never be among the codes it is kept from restarting on.+ */+export const EXIT_RESTARTING = 7;/*** The exit code a command that threw `error` ends with: configuration errors,diff --git a/console/packages/agent-controller/src/main.test.ts b/console/packages/agent-controller/src/main.test.tsindex 6e1ddcbb3..f179a32d0 100644--- a/console/packages/agent-controller/src/main.test.ts+++ b/console/packages/agent-controller/src/main.test.ts@@ -1,11 +1,12 @@-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import { EXIT_CONFIGURATION, EXIT_FAILURE, EXIT_OK } from './exit-codes';-import { infoChange, main } from './main';+import { infoChange, installerFor, isSupervised, main, planServiceReleases } from './main';import { dataLayout } from './paths';import { CONTROLLER_CREDENTIAL, FileSecretStore } from './secrets';+import { ReleasesDirInstaller } from './self-update';import { ControllerStore } from './store';let dir: string;@@ -282,3 +283,98 @@ describe('infoChange', () => {expect(() => infoChange({ description: 'y'.repeat(501) })).toThrow(/at most 500/);});});++describe('isSupervised', () => {+ it('restarts on 7 under the separate-users service, launchd, or anything systemd runs', () => {+ expect(isSupervised(null, false, {})).toBe(false);+ expect(+ isSupervised(+ {+ v: 1,+ user: 'builder',+ uid: 1001,+ gid: 990,+ dataDir: '/var/lib/controller',+ agentsDir: '/var/lib/switch-agents/1001',+ agentUsers: 4,+ node: '/usr/bin/node',+ bundle: '/opt/shared-host.mjs',+ releasesDir: null,+ },+ false,+ {}+ )+ ).toBe(true);+ expect(isSupervised(null, true, {})).toBe(true);+ expect(isSupervised(null, false, { INVOCATION_ID: 'abc' })).toBe(true);+ });+});++const NPM = { run: vi.fn(), node: 'node', cacheDir: 'cache', env: {} };++describe('installerFor', () => {+ let root: string;+ beforeEach(() => {+ root = mkdtempSync(join(tmpdir(), 'installer-for-'));+ });+ afterEach(() => rmSync(root, { recursive: true, force: true }));++ it('says a Windows controller does not update itself, whatever it runs from', async () => {+ const installer = await installerFor(null, null, NPM, 'win32');+ expect(installer.unavailable).toMatch(/Linux and macOS only/);+ });++ it.skipIf(process.platform === 'win32')(+ 'installs into the releases directory a service runs through',+ async () => {+ const releasesDir = join(root, 'releases');+ mkdirSync(releasesDir);+ expect((await installerFor(null, releasesDir, NPM, 'linux')).unavailable).toMatch(+ /no current link/+ );+ symlinkSync(join(root, 'prefix'), join(releasesDir, 'current'));+ const installer = await installerFor(null, releasesDir, NPM, 'linux');+ expect(installer.unavailable).toBeNull();+ expect(installer).toBeInstanceOf(ReleasesDirInstaller);+ }+ );+});++describe.skipIf(process.platform === 'win32')('planServiceReleases', () => {+ let root: string;+ beforeEach(() => {+ root = mkdtempSync(join(tmpdir(), 'service-releases-'));+ });+ afterEach(() => rmSync(root, { recursive: true, force: true }));++ const cliIn = (prefix: string) =>+ join(prefix, 'lib', 'node_modules', 'switch-agent-controller', 'dist', 'cli.js');++ it('runs an npm install through <data dir>/releases', async () => {+ const dir = join(root, 'data', 'releases');+ expect(await planServiceReleases(cliIn('/home/me/.local'), dir)).toEqual({+ dir,+ prefix: '/home/me/.local',+ });+ });++ it('has none for a checkout', async () => {+ expect(await planServiceReleases('/src/switch/console/dist/cli.js', root)).toBeNull();+ });++ it('runs again from a release of its own directory, and refuses another setup’s', async () => {+ const own = join(root, 'data', 'releases');+ const other = join(root, 'var', 'releases');+ for (const dir of [own, other]) {+ mkdirSync(join(dir, '0.31.0'), { recursive: true });+ symlinkSync(join(dir, '0.31.0'), join(dir, 'current'));+ }+ expect(await planServiceReleases(cliIn(join(own, '0.31.0')), own)).toEqual({+ dir: own,+ prefix: join(own, '0.31.0'),+ });+ await expect(planServiceReleases(cliIn(join(other, '0.31.0')), own)).rejects.toThrow(+ /releases directory of another setup/+ );+ });+});diff --git a/console/packages/agent-controller/src/main.ts b/console/packages/agent-controller/src/main.tsindex 6d0c64548..92160d4d5 100644--- a/console/packages/agent-controller/src/main.ts+++ b/console/packages/agent-controller/src/main.ts@@ -1,7 +1,7 @@-import { spawn } from 'node:child_process';import { realpathSync } from 'node:fs';+import { rm } from 'node:fs/promises';import { hostname } from 'node:os';-import { join, resolve } from 'node:path';+import { dirname, join, resolve } from 'node:path';import { parseArgs } from 'node:util';import {cliToolsBase,@@ -30,6 +30,7 @@ import {EXIT_TAKEN_OVER,EXIT_UPGRADE_REQUIRED,EXIT_CREDENTIAL_INVALID,+ EXIT_RESTARTING,exitCodeFor,isParseArgsError,} from './exit-codes';@@ -44,6 +45,7 @@ import { createLogger, errorMessage } from './log';import { loginShellPath } from './login-path';import { dataLayout, ensureDataDir, resolveDataDir, serverWorkspacesDir } from './paths';import { definitionProblem } from './reconcile';+import { isManagedRelease, readCurrent, throughCurrent } from './releases';import { emptyObservation, InProcessRuntime, observeOnDisk, probeProvider } from './runtime';import { AgentRuntimes } from './runtimes';import { type ControllerInfoChange, PROTOCOL_VERSION } from './schemas';@@ -57,9 +59,23 @@ import {SECRET_STORE_KINDS,secretStoreFor,} from './secrets';+import {+ downloadPackage,+ downloadVerified,+ MAX_PACKAGE_BYTES,+ type NpmSetup,+ npmInstall,+ PrefixInstaller,+ type ReleaseInstaller,+ ReleasesDirInstaller,+ runInstallCommand,+ SelfUpdater,+ unavailableInstaller,+} from './self-update';import {assertControllerCanRunSeparateUsers,DEFAULT_AGENT_USERS,+ ensureReleases,findSeparateUsersConfig,installSeparateUsers,loadSeparateUsersConfig,@@ -99,6 +115,7 @@ Commands:run [--data-dir <dir>] [--shared-host-bundle <path>][--controller-id <id> --server <agent-bridge-url> [--name <name>]][--credential-stdin] [--agent-runtime default|separate-user]+ [--releases-dir <dir>]Run the agents assigned to this machine and report their status.--agent-runtime separate-user runs every agent as a Linux user of itsown, as set up by install-service --separate-users.@@ -106,6 +123,8 @@ Commands:the data directory holds none, and move the same identity to a newserver URL when it holds that one. --credential-stdin reads thecontroller credential from stdin and keeps it in memory only.+ --releases-dir, which install-service passes, is where it installs+ the releases Switch names, beside the ones before.set-info [--name <name>] [--description <text>] [--data-dir <dir>]Rename this machine and/or change its description on the server, withthis controller's own credential. Either or both; --description ""@@ -116,11 +135,15 @@ Commands:Run this controller as a service of this user: a systemd user unit onLinux, a launchd agent on macOS. It starts now and again at each login(at boot too, on Linux with lingering on), and restarts after an error- that may pass. It keeps the PATH of the shell that installed it.+ that may pass. It keeps the PATH of the shell that installed it. It+ runs the controller through <data dir>/releases/current, where it+ installs the releases Switch names and keeps the one before.uninstall-service [--data-dir <dir>]- Stop the service and remove it. The enrollment and data are kept.+ Stop the service and remove it, and the releases it installed. The+ enrollment and data are kept.install-service --separate-users [--user <name>] [--data-dir <dir>][--env-file <path>] [--agents-dir <dir>] [--agent-users <n>] [--no-block]+ [--releases-dir <dir>]As root, once (Linux with systemd and polkit): run every agent as aLinux user of its own, seeing only its own directory. Makes <n> agentusers (default 16), a unit template for them, a polkit rule that lets@@ -128,7 +151,10 @@ Commands:system service of --user (default: the user who ran sudo). Node and thecontroller must be installed system-wide. Run it again to change it.--no-block queues the controller's start rather than waiting for it,- for a service the controller's unit is ordered after.+ for a service the controller's unit is ordered after. --releases-dir+ (default /var/lib/switch-agent-controller-<uid>/releases) is where the+ controller installs the releases Switch names; the units run it+ through <dir>/current.uninstall-service --separate-users [--user <name>]As root: stop the controller and its agents, and remove what the setupmade. The agents' directories are kept.@@ -139,6 +165,9 @@ Commands:Install the newest release with npm, and restart the service if it runs.--check only says whether there is one.+When Switch names a release for this controller, run installs it once no+session runs and exits 7 for its service to start it again on it.+run, install-service and doctor --env-file read NAME=value lines (as forsystemd's EnvironmentFile) into the agents' environment: provider API keys, orVertex AI and Bedrock settings such as CLAUDE_CODE_USE_VERTEX,@@ -154,7 +183,8 @@ the one built in the workspace.Log level: SWITCH_CONTROLLER_LOG_LEVEL (debug, info, warn, error; default info).Exit codes: 0 stopped, 1 error that may pass, 2 configuration error,3 revoked, 4 taken over by another instance, 5 the server needs a newer-controller, 6 the server knows no controller by this credential.+controller, 6 the server knows no controller by this credential, 7 restarting+into an update (start it again).`;const EXIT_CODE_FOR: Record<ControllerExit, number> = {@@ -163,6 +193,7 @@ const EXIT_CODE_FOR: Record<ControllerExit, number> = {taken_over: EXIT_TAKEN_OVER,upgrade_required: EXIT_UPGRADE_REQUIRED,credential_invalid: EXIT_CREDENTIAL_INVALID,+ restarting: EXIT_RESTARTING,};function bundlePath(flag: string | undefined): string {@@ -343,12 +374,18 @@ async function runCommand(args: string[]): Promise<number> {'env-file': { type: 'string' },launchd: { type: 'boolean' },'agent-runtime': { type: 'string' },+ 'releases-dir': { type: 'string' },},strict: true,});const agentRuntime = values['agent-runtime'] ?? 'default';if (agentRuntime !== 'default' && agentRuntime !== 'separate-user')throw new UsageError('--agent-runtime must be default or separate-user.');+ if (agentRuntime === 'separate-user' && values['releases-dir'] !== undefined)+ throw new UsageError(+ '--releases-dir is for a service of your user; agents as users of their own read it from their setup.'+ );+ const releasesDir = values['releases-dir'] ? resolve(values['releases-dir']) : null;// Before anything reads the environment: the agents inherit it.const fromFile = values['env-file'] ? await readEnvFile(resolve(values['env-file'])) : {};Object.assign(process.env, fromFile);@@ -408,9 +445,36 @@ async function runCommand(args: string[]): Promise<number> {file: values['env-file'],names: Object.keys(fromFile),});- // A controller a parent runs is updated with the parent.+ // A controller a parent runs (Switch Console's own) is updated with the parent.if (credential === null) void announceUpdate(log);try {+ const selfUpdate =+ credential === null+ ? new SelfUpdater({+ version: VERSION,+ installer: await installerFor(+ separateUsers,+ releasesDir,+ updateNpm(dataDir),+ process.platform+ ),+ download: (release, signal) =>+ downloadVerified({+ fetch,+ release,+ dir: updatesDir(dataDir),+ maxBytes: MAX_PACKAGE_BYTES,+ signal,+ }),+ attempts: {+ last: () => store.updateAttempt(),+ save: (attempt) => store.saveUpdateAttempt(attempt),+ },+ supervised: isSupervised(separateUsers, values.launchd === true, process.env),+ log,+ now: Date.now,+ })+ : null;if (controllerId !== undefined && values.server !== undefined) {const previousServer = store.identity()?.server;const adopted = adoptIdentity(@@ -467,6 +531,7 @@ async function runCommand(args: string[]): Promise<number> {// no directory works in a folder of its own there.workspacesFor: separateUsers ? () => unitAgentRoot(separateUsers) : serverWorkspacesDir,version: VERSION,+ selfUpdate,now: Date.now,random: Math.random,timing: DEFAULT_TIMING,@@ -590,6 +655,72 @@ async function announceUpdate(log: ReturnType<typeof createLogger>): Promise<voi}}+/** Where downloaded releases and npm's cache go: the controller's own data directory. */+function updatesDir(dataDir: string): string {+ return join(dataDir, 'updates');+}++function updateNpm(dataDir: string): NpmSetup {+ return {+ run: runInstallCommand,+ node: process.execPath,+ cacheDir: join(updatesDir(dataDir), 'npm-cache'),+ env: process.env,+ };+}++/**+ * Whether exiting with `EXIT_RESTARTING` starts this controller again: the+ * separate-users runtime only runs as its system service, `--launchd` is the+ * launchd agent, and systemd sets `INVOCATION_ID` for what it runs (whose+ * `Restart=on-failure` restarts a non-zero exit).+ */+export function isSupervised(+ separateUsers: SeparateUsersConfig | null,+ launchd: boolean,+ env: NodeJS.ProcessEnv+): boolean {+ return separateUsers !== null || launchd || Boolean(env.INVOCATION_ID);+}++/** How this controller installs a release for itself, as it was installed. */+export async function installerFor(+ separateUsers: SeparateUsersConfig | null,+ releasesDir: string | null,+ npm: NpmSetup,+ platform: NodeJS.Platform+): Promise<ReleaseInstaller> {+ if (platform === 'win32')+ return unavailableInstaller(+ 'This controller updates itself on Linux and macOS only. On Windows, install the release Switch names by hand and start the controller again.'+ );+ if (separateUsers)+ return separateUsers.releasesDir+ ? new ReleasesDirInstaller({ npm, releasesDir: separateUsers.releasesDir })+ : unavailableInstaller(+ 'The setup for agents as users of their own predates releases directories; run sudo switch-agent-controller install-service --separate-users again so this controller can update itself.'+ );+ if (releasesDir) {+ const current = await readCurrent(releasesDir);+ return current.kind === 'link'+ ? new ReleasesDirInstaller({ npm, releasesDir })+ : unavailableInstaller(+ `--releases-dir ${releasesDir} has no current link, which install-service makes; run switch-agent-controller install-service again so this controller can update itself.`+ );+ }+ const cli = cliPath();+ const prefix = installPrefix(cli);+ if (!prefix)+ return unavailableInstaller(+ `This controller runs from ${cli}, not an npm install (a checkout?), so it does not update itself.`+ );+ if (await isManagedRelease(prefix))+ return unavailableInstaller(+ `This controller runs from ${prefix}, a releases directory, but without --releases-dir, which the service install-service sets up passes; only that service installs releases there.`+ );+ return new PrefixInstaller({ npm, prefix, cli });+}+/** The CLI file this process runs, through any npm bin link, as a service names it. */function cliPath(): string {const entry = process.argv[1];@@ -609,11 +740,12 @@ async function installServiceCommand(args: string[]): Promise<number> {'agents-dir': { type: 'string' },'agent-users': { type: 'string' },'no-block': { type: 'boolean' },+ 'releases-dir': { type: 'string' },},strict: true,});if (values['separate-users']) return installSeparateUsersCommand(values);- for (const flag of ['user', 'agents-dir', 'agent-users', 'no-block'] as const)+ for (const flag of ['user', 'agents-dir', 'agent-users', 'no-block', 'releases-dir'] as const)if (values[flag] !== undefined)throw new UsageError(`--${flag} is for install-service --separate-users.`);const envFile = values['env-file'] ? resolve(values['env-file']) : null;@@ -631,21 +763,60 @@ async function installServiceCommand(args: string[]): Promise<number> {} finally {store.close();}+ const cli = cliPath();+ const releases = await planServiceReleases(cli, serviceReleasesDir(dataDir));+ const releasesNote = releases+ ? await ensureReleases({+ releasesDir: releases.dir,+ prefix: releases.prefix,+ version: VERSION,+ owner: null,+ })+ : `This controller runs from ${cli}, not an npm install (a checkout?), so it does not update itself.`;const report = await installService({node: process.execPath,- cli: cliPath(),+ cli: releases ? throughCurrent(releases.dir, releases.prefix, cli) : cli,dataDir,envFile,- sharedHostBundle,+ sharedHostBundle:+ releases && sharedHostBundle+ ? throughCurrent(releases.dir, releases.prefix, sharedHostBundle)+ : sharedHostBundle,+ releasesDir: releases?.dir ?? null,path: process.env.PATH ?? '/usr/local/bin:/usr/bin:/bin',},runSecretCommand);- process.stdout.write([`Installed and started: ${report.file}`, ...report.notes, ''].join('\n'));+ process.stdout.write(+ [`Installed and started: ${report.file}`, ...report.notes, releasesNote, ''].join('\n')+ );return EXIT_OK;}+/** Where a service of the user's keeps the releases it installs for itself. */+function serviceReleasesDir(dataDir: string): string {+ return join(dataDir, 'releases');+}++/**+ * The releases directory a service of the user's runs through, for the+ * controller at `cli`, and the npm prefix that controller came from; null+ * when it is not an npm install (a checkout), which cannot have one.+ */+export async function planServiceReleases(+ cli: string,+ dir: string+): Promise<{ dir: string; prefix: string } | null> {+ const prefix = installPrefix(cli);+ if (!prefix) return null;+ if ((await isManagedRelease(prefix)) && resolve(dirname(prefix)) !== resolve(dir))+ throw new ConfigurationError(+ `This controller runs from ${prefix}, the releases directory of another setup. Run install-service with the controller npm installed (switch-agent-controller on your PATH).`+ );+ return { dir, prefix };+}+async function installSeparateUsersCommand(values: {'data-dir'?: string;'env-file'?: string;@@ -654,6 +825,7 @@ async function installSeparateUsersCommand(values: {'agents-dir'?: string;'agent-users'?: string;'no-block'?: boolean;+ 'releases-dir'?: string;}): Promise<number> {const user = values.user ?? process.env.SUDO_USER;if (!user)@@ -675,6 +847,8 @@ async function installSeparateUsersCommand(values: {node: process.execPath,cli: cliPath(),bundle: bundlePath(values['shared-host-bundle']),+ version: VERSION,+ releasesDir: values['releases-dir'] ? resolve(values['releases-dir']) : null,envFile,path: process.env.PATH ?? '/usr/local/bin:/usr/bin:/bin',noBlock: values['no-block'] === true,@@ -709,9 +883,19 @@ async function uninstallServiceCommand(args: string[]): Promise<number> {}if (values.user !== undefined)throw new UsageError('--user is for uninstall-service --separate-users.');- const removed = await uninstallService(resolveDataDir(values['data-dir']), runSecretCommand);+ const dataDir = resolveDataDir(values['data-dir']);+ const removed = await uninstallService(dataDir, runSecretCommand);+ const releasesDir = serviceReleasesDir(dataDir);+ const current = await readCurrent(releasesDir);+ if (current.kind !== 'missing') await rm(releasesDir, { recursive: true, force: true });process.stdout.write(- removed ? 'Stopped and removed the service.\n' : 'No service was installed.\n'+ [+ removed ? 'Stopped and removed the service.' : 'No service was installed.',+ ...(current.kind !== 'missing'+ ? [`Removed the releases it installed for itself, in ${releasesDir}.`]+ : []),+ '',+ ].join('\n'));return EXIT_OK;}@@ -799,24 +983,34 @@ async function updateCommand(args: string[]): Promise<number> {);return EXIT_OK;}+ const prefix = installPrefix(cliPath());+ if (prefix && (await isManagedRelease(prefix)))+ throw new ConfigurationError(+ `This controller runs from ${prefix}, a releases directory install-service set up: it installs the release Switch names itself, once no session runs. To install another by hand, install it with npm and run install-service again.`+ );+ const dataDir = resolveDataDir(values['data-dir']);process.stdout.write(`Installing ${latest.version} from ${latest.packageUrl}\n`);- const code = await new Promise<number | null>((done, fail) => {- const prefix = installPrefix(cliPath());- const npmArgs = [- 'install',- '--global',- ...(prefix ? ['--prefix', prefix] : []),- latest.packageUrl,- ];- const child = spawn('npm', npmArgs, { stdio: 'inherit' });- child.once('error', fail);- child.once('exit', done);+ const { file } = await downloadPackage({+ fetch,+ url: latest.packageUrl,+ dir: updatesDir(dataDir),+ maxBytes: MAX_PACKAGE_BYTES,+ signal: null,});- if (code !== 0)+ try {+ await npmInstall(updateNpm(dataDir), prefix, file, null);+ } catch (error) {throw new Error(- `npm install exited with ${code}. If it could not write where the controller is installed, install it again with the installer, which falls back to ~/.local.`+ `${errorMessage(error)}\nIf npm could not write where the controller is installed, install it again with the installer, which falls back to ~/.local.`+ );+ } finally {+ await rm(file, { force: true });+ }+ const releasesDir = serviceReleasesDir(dataDir);+ if (prefix && (await readCurrent(releasesDir)).kind === 'link')+ process.stdout.write(+ `${await ensureReleases({ releasesDir, prefix, version: latest.version, owner: null })}\n`);- const dataDir = resolveDataDir(values['data-dir']);if ((await serviceState(dataDir, runSecretCommand)) === 'running') {await restartService(dataDir, runSecretCommand);process.stdout.write(`Installed ${latest.version} and restarted the service.\n`);diff --git a/console/packages/agent-controller/src/releases.test.ts b/console/packages/agent-controller/src/releases.test.tsnew file mode 100644index 000000000..8f020bba8--- /dev/null+++ b/console/packages/agent-controller/src/releases.test.ts@@ -0,0 +1,50 @@+import { mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, utimesSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { describe, expect, it } from 'vitest';+import { pruneReleases, throughCurrent } from './releases';++describe.skipIf(process.platform === 'win32')('throughCurrent', () => {+ it('reaches a file of the release through current, and leaves one outside it alone', () => {+ expect(+ throughCurrent(+ '/data/releases',+ '/home/me/.local',+ '/home/me/.local/lib/node_modules/switch-agent-controller/dist/cli.js'+ )+ ).toBe('/data/releases/current/lib/node_modules/switch-agent-controller/dist/cli.js');+ expect(throughCurrent('/data/releases', '/home/me/.local', '/opt/shared-host.mjs')).toBe(+ '/opt/shared-host.mjs'+ );+ expect(throughCurrent('/data/releases', '/home/me/.local', '/home/me/.localother/x')).toBe(+ '/home/me/.localother/x'+ );+ });+});++describe.skipIf(process.platform === 'win32')('pruneReleases', () => {+ it('keeps dev builds as releases, and clears what an interrupted install of one left', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'prune-'));+ try {+ const made = [+ '0.31.1-dev.3.gaaaaaaa',+ '0.31.1-dev.4.gbbbbbbb',+ '0.31.1-dev.5.gccccccc',+ '0.31.1-dev.6.gddddddd.partial',+ ];+ made.forEach((name, index) => {+ mkdirSync(join(dir, name));+ utimesSync(join(dir, name), 1000 + index, 1000 + index);+ });+ symlinkSync(join(dir, '0.31.1-dev.5.gccccccc'), join(dir, 'current'));+ await pruneReleases(dir, []);+ expect(readdirSync(dir).sort()).toEqual([+ '0.31.1-dev.4.gbbbbbbb',+ '0.31.1-dev.5.gccccccc',+ 'current',+ ]);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ });+});diff --git a/console/packages/agent-controller/src/releases.ts b/console/packages/agent-controller/src/releases.tsnew file mode 100644index 000000000..2597d2094--- /dev/null+++ b/console/packages/agent-controller/src/releases.ts@@ -0,0 +1,125 @@+import { lchown, lstat, readdir, readFile, readlink, rename, rm, symlink } from 'node:fs/promises';+import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';+import { RELEASE_VERSION } from './update';++/**+ * A releases directory: where a controller run by `install-service` keeps the+ * releases it installs for itself (`<data dir>/releases` for a service of+ * the user's, its own directory with `--separate-users`). Each+ * `<releases>/<version>/` is an npm prefix holding one release, and+ * `<releases>/current` links to the prefix the controller's unit and its+ * agents' units run from: one of those, or the one the controller was first+ * installed in (a machine image's, say), which stays outside it.+ */++/** The package name a release installs as (`scripts/package.mjs`). */+export const PACKAGE_NAME = 'switch-agent-controller';+export const CURRENT = 'current';+const CURRENT_NEXT = 'current.new';++/** Where an npm prefix holds the release's files. */+export function packageDir(prefix: string): string {+ return join(prefix, 'lib', 'node_modules', PACKAGE_NAME);+}++/** The command npm links into an npm prefix for the release. */+export function prefixBin(prefix: string): string {+ return join(prefix, 'bin', PACKAGE_NAME);+}++export function currentLink(releasesDir: string): string {+ return join(releasesDir, CURRENT);+}++/** `path`, a file of the release in the npm prefix `prefix`, as reached through `<releases>/current`; a path outside `prefix` as it is. */+export function throughCurrent(releasesDir: string, prefix: string, path: string): string {+ const inside = relative(prefix, path);+ if (inside === '' || inside.startsWith('..') || isAbsolute(inside)) return path;+ return join(currentLink(releasesDir), inside);+}++/** What `<releases>/current` is now. */+export type CurrentRelease =+ | { kind: 'missing' }+ | { kind: 'not_a_link' }+ /** `version` is null when the link is dangling or what it names holds no readable release. */+ | { kind: 'link'; target: string; version: string | null };++export async function readCurrent(releasesDir: string): Promise<CurrentRelease> {+ const link = currentLink(releasesDir);+ try {+ if (!(await lstat(link)).isSymbolicLink()) return { kind: 'not_a_link' };+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { kind: 'missing' };+ throw error;+ }+ const named = await readlink(link);+ const target = isAbsolute(named) ? named : resolve(releasesDir, named);+ return { kind: 'link', target, version: await installedVersion(target) };+}++/** The version of the release installed in the npm prefix `prefix`; null when none is readable there. */+export async function installedVersion(prefix: string): Promise<string | null> {+ try {+ const manifest = JSON.parse(await readFile(join(packageDir(prefix), 'package.json'), 'utf8'));+ return typeof manifest?.version === 'string' ? manifest.version : null;+ } catch {+ return null;+ }+}++/**+ * Points `<releases>/current` at `target` in one step: a new link beside it,+ * renamed over it, so a unit starting meanwhile sees the old one or the new.+ */+export async function pointCurrent(+ releasesDir: string,+ target: string,+ owner: { uid: number; gid: number } | null+): Promise<void> {+ const next = join(releasesDir, CURRENT_NEXT);+ await rm(next, { force: true });+ await symlink(target, next);+ if (owner) await lchown(next, owner.uid, owner.gid);+ await rename(next, currentLink(releasesDir));+}++/**+ * Whether the npm prefix `prefix` is one release of a releases directory,+ * `<releases>/<version>` beside a `current` link, which only the controller+ * that owns it installs into.+ */+export async function isManagedRelease(prefix: string): Promise<boolean> {+ if (!RELEASE_VERSION.test(basename(prefix))) return false;+ try {+ return (await lstat(currentLink(dirname(prefix)))).isSymbolicLink();+ } catch {+ return false;+ }+}++/**+ * Removes the releases no longer needed: every version directory but those in+ * `keep` and the most recently installed other one, and what an interrupted+ * install left behind. Only entries of `releasesDir` are ever removed, so a+ * prefix outside it that `current` once named is never touched.+ */+export async function pruneReleases(releasesDir: string, keep: string[]): Promise<string[]> {+ const kept = new Set(keep.map((path) => resolve(path)));+ const current = await readCurrent(releasesDir);+ if (current.kind === 'link') kept.add(resolve(current.target));+ const spare: { path: string; mtimeMs: number }[] = [];+ const leftovers: string[] = [];+ for (const entry of await readdir(releasesDir)) {+ const path = join(releasesDir, entry);+ if (entry === CURRENT || kept.has(path)) continue;+ // Before the version test: `0.31.0-dev.4.partial` reads as a version too.+ if (entry === CURRENT_NEXT || /\.(partial|replaced-[\w-]+)$/.test(entry)) leftovers.push(path);+ else if (RELEASE_VERSION.test(entry))+ spare.push({ path, mtimeMs: (await lstat(path)).mtimeMs });+ }+ spare.sort((a, b) => b.mtimeMs - a.mtimeMs);+ const removed = [...spare.slice(1).map((entry) => entry.path), ...leftovers];+ for (const path of removed) await rm(path, { recursive: true, force: true });+ return removed;+}diff --git a/console/packages/agent-controller/src/schemas.test.ts b/console/packages/agent-controller/src/schemas.test.tsindex 38d7b5716..3fa997a7b 100644--- a/console/packages/agent-controller/src/schemas.test.ts+++ b/console/packages/agent-controller/src/schemas.test.ts@@ -224,4 +224,32 @@ describe('status report', () => {const { reason: _reason, ...agent } = report.agents[0]!;expect(statusReportSchema.safeParse({ ...report, agents: [agent] }).success).toBe(false);});++ it('says why the controller is not on the release Switch names, when it is not', () => {+ expect(+ statusReportSchema.safeParse({+ ...report,+ update: { target: '0.31.0', state: 'failed', problem: 'npm install failed' },+ }).success+ ).toBe(true);+ expect(+ statusReportSchema.safeParse({ ...report, update: { target: '0.31.0', state: 'done' } })+ .success+ ).toBe(false);+ });+});++describe('status response', () => {+ it('keeps controller_release as sent, absent included, whatever its shape', () => {+ const base = { assignment_revision: 3, report_within_s: 60 };+ expect(statusResponseSchema.parse(base).controller_release).toBeUndefined();+ expect(+ statusResponseSchema.parse({ ...base, controller_release: null }).controller_release+ ).toBeNull();+ const release = { version: '0.31.0', url: 'https://example.invalid/a.tgz', sha256: 'f' };+ expect(+ statusResponseSchema.parse({ ...base, controller_release: release }).controller_release+ ).toEqual(release);+ expect(statusResponseSchema.safeParse({ ...base, controller_release: 42 }).success).toBe(true);+ });});diff --git a/console/packages/agent-controller/src/schemas.ts b/console/packages/agent-controller/src/schemas.tsindex a007bef9c..5611f7e09 100644--- a/console/packages/agent-controller/src/schemas.ts+++ b/console/packages/agent-controller/src/schemas.ts@@ -264,12 +264,26 @@ export const statusReportSchema = z.object({providers: z.array(providerStatusSchema),tools: z.array(toolStatusSchema),agents: z.array(agentStatusSchema),+ /** Why this controller does not run the release Switch names, when it does not. */+ update: z+ .object({+ target: z.string().min(1),+ state: z.enum(['waiting_idle', 'failed', 'restart_needed']),+ problem: z.string().max(500).optional(),+ })+ .optional(),});export type StatusReport = z.infer<typeof statusReportSchema>;export const statusResponseSchema = z.object({assignment_revision: revision,report_within_s: z.number().int().positive(),+ /**+ * The release Switch wants this controller to run, or null/absent for none.+ * Kept unread here so a malformed one cannot fail the whole response:+ * `readControllerRelease` reads it.+ */+ controller_release: z.unknown().optional(),});export type StatusResponse = z.infer<typeof statusResponseSchema>;diff --git a/console/packages/agent-controller/src/self-update.test.ts b/console/packages/agent-controller/src/self-update.test.tsnew file mode 100644index 000000000..f66932f78--- /dev/null+++ b/console/packages/agent-controller/src/self-update.test.ts@@ -0,0 +1,535 @@+import { createHash } from 'node:crypto';+import {+ existsSync,+ mkdirSync,+ mkdtempSync,+ readdirSync,+ readFileSync,+ readlinkSync,+ rmSync,+ symlinkSync,+ utimesSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { afterEach, beforeEach, describe, expect, it } from 'vitest';+import { silentLogger } from './log';+import { packageDir } from './releases';+import {+ type ControllerReleaseTarget,+ downloadVerified,+ type InstallRunner,+ type NpmSetup,+ PrefixInstaller,+ readControllerRelease,+ type ReleaseInstaller,+ ReleasesDirInstaller,+ RETRY_AFTER_MS,+ SelfUpdater,+ type UpdateAttempt,+ type UpdateHost,+ unavailableInstaller,+} from './self-update';++let dir: string;++beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), 'self-update-'));+});++afterEach(() => {+ rmSync(dir, { recursive: true, force: true });+});++const SHA = 'a'.repeat(64);++describe('readControllerRelease', () => {+ it('is none when absent or null', () => {+ expect(readControllerRelease(undefined)).toEqual({ release: null, problem: null });+ expect(readControllerRelease(null)).toEqual({ release: null, problem: null });+ });++ it('reads a release, with its SHA-256 in lower case', () => {+ expect(+ readControllerRelease({+ version: '0.31.0',+ url: 'https://example.invalid/switch-agent-controller-0.31.0.tgz',+ sha256: SHA.toUpperCase(),+ extra: 'ignored',+ })+ ).toEqual({+ release: {+ version: '0.31.0',+ url: 'https://example.invalid/switch-agent-controller-0.31.0.tgz',+ sha256: SHA,+ },+ problem: null,+ });+ });++ it("reads a dev build's version", () => {+ const read = readControllerRelease({+ version: '0.31.1-dev.412.gabc1234',+ url: 'https://example.invalid/switch-agent-controller-0.31.1-dev.412.gabc1234.tgz',+ sha256: SHA,+ });+ expect(read.problem).toBeNull();+ expect(read.release?.version).toBe('0.31.1-dev.412.gabc1234');+ });++ it('treats a malformed one as none, and says why', () => {+ for (const raw of [+ 'latest',+ { version: '0.31', url: 'https://example.invalid/a.tgz', sha256: SHA },+ { version: '0.31.0', url: 'ftp://example.invalid/a.tgz', sha256: SHA },+ { version: '0.31.0', url: 'https://example.invalid/a.tgz', sha256: 'abc' },+ { version: '../../etc', url: 'https://example.invalid/a.tgz', sha256: SHA },+ { version: '0.31.0-dev..1', url: 'https://example.invalid/a.tgz', sha256: SHA },+ { version: '0.31.0-dev/../x', url: 'https://example.invalid/a.tgz', sha256: SHA },+ ]) {+ const read = readControllerRelease(raw);+ expect(read.release).toBeNull();+ expect(read.problem).toBeTruthy();+ }+ });+});++function serving(body: Buffer, headers: Record<string, string> = {}): typeof fetch {+ return (async () => new Response(new Uint8Array(body), { status: 200, headers })) as typeof fetch;+}++/** A body sent in chunks, with no Content-Length to go by. */+function streaming(chunks: Buffer[]): typeof fetch {+ return (async () =>+ new Response(+ new ReadableStream({+ start(controller) {+ for (const chunk of chunks) controller.enqueue(new Uint8Array(chunk));+ controller.close();+ },+ }),+ { status: 200 }+ )) as typeof fetch;+}++function releaseOf(body: Buffer, version = '0.31.0'): ControllerReleaseTarget {+ return {+ version,+ url: `https://example.invalid/switch-agent-controller-${version}.tgz`,+ sha256: createHash('sha256').update(body).digest('hex'),+ };+}++describe('downloadVerified', () => {+ it('keeps a package whose SHA-256 is the one Switch named', async () => {+ const body = Buffer.from('a package');+ const file = await downloadVerified({+ fetch: serving(body),+ release: releaseOf(body),+ dir: join(dir, 'updates'),+ maxBytes: 1024,+ signal: null,+ });+ expect(readFileSync(file)).toEqual(body);+ expect(dirname(file)).toBe(join(dir, 'updates'));+ });++ it('deletes a package whose SHA-256 differs', async () => {+ const body = Buffer.from('a package');+ await expect(+ downloadVerified({+ fetch: serving(Buffer.from('another package')),+ release: releaseOf(body),+ dir: join(dir, 'updates'),+ maxBytes: 1024,+ signal: null,+ })+ ).rejects.toThrow(/not the [0-9a-f]{64} Switch named/);+ expect(readdirSync(join(dir, 'updates'))).toEqual([]);+ });++ it('refuses a package larger than the cap, declared or sent', async () => {+ const body = Buffer.alloc(2048, 1);+ await expect(+ downloadVerified({+ fetch: serving(body, { 'content-length': '2048' }),+ release: releaseOf(body),+ dir: join(dir, 'updates'),+ maxBytes: 1024,+ signal: null,+ })+ ).rejects.toThrow(/2048 bytes, more than the 1024/);+ await expect(+ downloadVerified({+ fetch: streaming([body.subarray(0, 1000), body.subarray(1000)]),+ release: releaseOf(body),+ dir: join(dir, 'updates'),+ maxBytes: 1024,+ signal: null,+ })+ ).rejects.toThrow(/more than the 1024 bytes/);+ expect(readdirSync(join(dir, 'updates'))).toEqual([]);+ });++ it('fails loud when the server refuses', async () => {+ const refusing = (async () => new Response('gone', { status: 404 })) as typeof fetch;+ await expect(+ downloadVerified({+ fetch: refusing,+ release: releaseOf(Buffer.from('x')),+ dir: join(dir, 'updates'),+ maxBytes: 1024,+ signal: null,+ })+ ).rejects.toThrow(/404/);+ });+});++/**+ * Stands in for npm and node: `install --global --prefix P <file>` installs+ * the release whose version the file holds into P; `node <cli> --version`+ * says the version of the release `cli` is in.+ */+function fakeNpm(options: { says?: string } = {}) {+ const calls: string[][] = [];+ const run: InstallRunner = async (file, args) => {+ calls.push([file, ...args]);+ if (args[0] === 'install') {+ const prefix = args[args.indexOf('--prefix') + 1]!;+ const version = readFileSync(args.at(-1)!, 'utf8').trim();+ const pkg = packageDir(prefix);+ mkdirSync(pkg, { recursive: true });+ writeFileSync(join(pkg, 'package.json'), JSON.stringify({ version }));+ writeFileSync(join(pkg, 'cli.mjs'), '');+ mkdirSync(join(prefix, 'bin'), { recursive: true });+ symlinkSync(+ '../lib/node_modules/switch-agent-controller/cli.mjs',+ join(prefix, 'bin', 'switch-agent-controller')+ );+ return { stdout: '' };+ }+ if (args[1] === '--version') {+ const cli = args[0]!;+ const pkg = cli.includes('/bin/') ? packageDir(dirname(dirname(cli))) : dirname(cli);+ const version = JSON.parse(readFileSync(join(pkg, 'package.json'), 'utf8')).version;+ return { stdout: `${options.says ?? version}\n` };+ }+ throw new Error(`Unexpected command ${file} ${args.join(' ')}`);+ };+ const setup: NpmSetup = {+ run,+ node: '/opt/node/bin/node',+ cacheDir: join(dir, 'npm-cache'),+ env: { PATH: '/usr/bin' },+ };+ return { setup, calls };+}++/** A downloaded package, as the fake npm reads it. */+function packageFile(version: string): string {+ const file = join(dir, `package-${version}.tgz`);+ writeFileSync(file, version);+ return file;+}++/** A release already in the releases directory, installed `ageS` seconds ago. */+function installed(releases: string, version: string, ageS: number): string {+ const prefix = join(releases, version);+ mkdirSync(packageDir(prefix), { recursive: true });+ writeFileSync(join(packageDir(prefix), 'package.json'), JSON.stringify({ version }));+ const at = new Date(Date.now() - ageS * 1000);+ utimesSync(prefix, at, at);+ return prefix;+}++describe.skipIf(process.platform === 'win32')('ReleasesDirInstaller', () => {+ it('installs beside the others, switches current, and keeps the previous release and one older', async () => {+ const releases = join(dir, 'releases');+ mkdirSync(releases);+ installed(releases, '0.28.0', 400);+ installed(releases, '0.29.0', 300);+ const previous = installed(releases, '0.30.0', 200);+ mkdirSync(join(releases, '0.27.0.partial'));+ symlinkSync(previous, join(releases, 'current'));+ const npm = fakeNpm();+ const installer = new ReleasesDirInstaller({ npm: npm.setup, releasesDir: releases });++ await installer.prepare('0.31.0', packageFile('0.31.0'), null);+ expect(readlinkSync(join(releases, 'current'))).toBe(previous);+ expect(existsSync(join(releases, '0.31.0.partial'))).toBe(false);+ await installer.activate('0.31.0');++ expect(readlinkSync(join(releases, 'current'))).toBe(join(releases, '0.31.0'));+ expect(readdirSync(releases).sort()).toEqual(['0.29.0', '0.30.0', '0.31.0', 'current']);+ expect(npm.calls[0]).toEqual([+ 'npm',+ 'install',+ '--global',+ '--prefix',+ join(releases, '0.31.0.partial'),+ join(dir, 'package-0.31.0.tgz'),+ ]);+ expect(npm.calls[1]).toEqual([+ '/opt/node/bin/node',+ join(releases, '0.31.0.partial', 'bin', 'switch-agent-controller'),+ '--version',+ ]);+ });++ it('never prunes a release outside the directory that current named', async () => {+ const releases = join(dir, 'releases');+ mkdirSync(releases);+ const baked = join(dir, 'opt', 'controller');+ mkdirSync(packageDir(baked), { recursive: true });+ symlinkSync(baked, join(releases, 'current'));+ const installer = new ReleasesDirInstaller({ npm: fakeNpm().setup, releasesDir: releases });+ await installer.prepare('0.31.0', packageFile('0.31.0'), null);+ await installer.activate('0.31.0');+ await installer.prepare('0.32.0', packageFile('0.32.0'), null);+ await installer.activate('0.32.0');+ expect(existsSync(baked)).toBe(true);+ expect(readdirSync(releases).sort()).toEqual(['0.31.0', '0.32.0', 'current']);+ });++ it('refuses a package that is not the version Switch named, leaving current as it was', async () => {+ const releases = join(dir, 'releases');+ mkdirSync(releases);+ const previous = installed(releases, '0.30.0', 10);+ symlinkSync(previous, join(releases, 'current'));+ const installer = new ReleasesDirInstaller({+ npm: fakeNpm({ says: '0.29.0' }).setup,+ releasesDir: releases,+ });+ await expect(installer.prepare('0.31.0', packageFile('0.31.0'), null)).rejects.toThrow(+ /says it is 0.29.0, not 0.31.0/+ );+ expect(readdirSync(releases).sort()).toEqual(['0.30.0', 'current']);+ expect(readlinkSync(join(releases, 'current'))).toBe(previous);+ });++ it('replaces a release of the same version installed before', async () => {+ const releases = join(dir, 'releases');+ mkdirSync(releases);+ const earlier = installed(releases, '0.31.0', 10);+ writeFileSync(join(earlier, 'stale'), '');+ const installer = new ReleasesDirInstaller({ npm: fakeNpm().setup, releasesDir: releases });+ await installer.prepare('0.31.0', packageFile('0.31.0'), null);+ await installer.activate('0.31.0');+ expect(existsSync(join(earlier, 'stale'))).toBe(false);+ expect(readdirSync(releases).sort()).toEqual(['0.31.0', 'current']);+ });+});++describe('PrefixInstaller', () => {+ it('installs over the prefix the controller came from, then checks the CLI there', async () => {+ const prefix = join(dir, 'local');+ const cli = join(packageDir(prefix), 'cli.mjs');+ const npm = fakeNpm();+ const installer = new PrefixInstaller({ npm: npm.setup, prefix, cli });+ await installer.prepare();+ expect(npm.calls).toEqual([]);+ await installer.activate('0.31.0', packageFile('0.31.0'));+ expect(npm.calls.map((call) => call.slice(1))).toEqual([+ ['install', '--global', '--prefix', prefix, join(dir, 'package-0.31.0.tgz')],+ [cli, '--version'],+ ]);+ });+});++/** An installer that records what it was asked, and fails where told. */+function recordingInstaller(fail: { prepare?: string; activate?: string } = {}) {+ const calls: string[] = [];+ const installer: ReleaseInstaller = {+ unavailable: null,+ prepare: async (version) => {+ calls.push(`prepare ${version}`);+ if (fail.prepare) throw new Error(fail.prepare);+ },+ activate: async (version) => {+ calls.push(`activate ${version}`);+ if (fail.activate) throw new Error(fail.activate);+ },+ };+ return { installer, calls };+}++function updater(+ overrides: Partial<ConstructorParameters<typeof SelfUpdater>[0]> & {+ installer: ReleaseInstaller;+ }+) {+ let saved: UpdateAttempt | null = null;+ const downloads: string[] = [];+ const clock = { now: 1_000_000 };+ const self = new SelfUpdater({+ version: '0.30.0',+ download: async (release) => {+ downloads.push(release.version);+ const file = join(dir, `download-${release.version}.tgz`);+ writeFileSync(file, '');+ return file;+ },+ attempts: { last: () => saved, save: (attempt) => void (saved = attempt) },+ supervised: true,+ log: silentLogger,+ now: () => clock.now,+ ...overrides,+ });+ return { self, downloads, clock, attempt: () => saved };+}++function host(idle: boolean | boolean[] = true) {+ const answers = Array.isArray(idle) ? [...idle] : null;+ let restarts = 0;+ const stop = new AbortController();+ const lent: UpdateHost = {+ idle: async () => (answers ? (answers.shift() ?? true) : (idle as boolean)),+ restart: () => void restarts++,+ signal: stop.signal,+ };+ return { lent, restarts: () => restarts, stop };+}++const named = {+ version: '0.31.0',+ url: 'https://example.invalid/switch-agent-controller-0.31.0.tgz',+ sha256: SHA,+};++describe('SelfUpdater', () => {+ it('leaves a controller alone when Switch names no release, or the one it runs', async () => {+ const { installer, calls } = recordingInstaller();+ const { self, downloads } = updater({ installer });+ const idle = host();+ await self.offer(undefined, idle.lent);+ await self.offer(null, idle.lent);+ await self.offer({ ...named, version: '0.30.0' }, idle.lent);+ await self.offer({ broken: true }, idle.lent);+ expect(downloads).toEqual([]);+ expect(calls).toEqual([]);+ expect(self.status()).toBeNull();+ expect(idle.restarts()).toBe(0);+ });++ it('installs another version while idle, older included, and restarts into it', async () => {+ const { installer, calls } = recordingInstaller();+ const { self, downloads } = updater({ installer });+ const idle = host();+ await self.offer({ ...named, version: '0.29.0' }, idle.lent);+ expect(downloads).toEqual(['0.29.0']);+ expect(calls).toEqual(['prepare 0.29.0', 'activate 0.29.0']);+ expect(idle.restarts()).toBe(1);+ expect(self.status()).toBeNull();+ expect(existsSync(join(dir, 'download-0.29.0.tgz'))).toBe(false);+ });++ it('waits while a session runs, and goes ahead at a later report once none does', async () => {+ const { installer, calls } = recordingInstaller();+ const { self, downloads, attempt } = updater({ installer });+ const busy = host(false);+ await self.offer(named, busy.lent);+ expect(downloads).toEqual([]);+ expect(self.status()).toEqual({ target: '0.31.0', state: 'waiting_idle' });+ expect(attempt()).toBeNull();+ const idle = host();+ await self.offer(named, idle.lent);+ expect(calls).toEqual(['prepare 0.31.0', 'activate 0.31.0']);+ expect(idle.restarts()).toBe(1);+ });++ it('checks again right before switching, and waits when a session started meanwhile', async () => {+ const { installer, calls } = recordingInstaller();+ const { self, attempt } = updater({ installer });+ const started = host([true, false]);+ await self.offer(named, started.lent);+ expect(calls).toEqual(['prepare 0.31.0']);+ expect(started.restarts()).toBe(0);+ expect(self.status()).toEqual({ target: '0.31.0', state: 'waiting_idle' });+ expect(attempt()).toBeNull();+ });++ it('reports a failure, and does not try the same release again within the hour', async () => {+ const { installer, calls } = recordingInstaller({ prepare: 'npm install failed: EACCES' });+ const { self, clock, downloads } = updater({ installer });+ const idle = host();+ await self.offer(named, idle.lent);+ expect(self.status()).toEqual({+ target: '0.31.0',+ state: 'failed',+ problem: 'npm install failed: EACCES',+ });+ clock.now += RETRY_AFTER_MS - 1;+ await self.offer(named, idle.lent);+ expect(downloads).toEqual(['0.31.0']);+ clock.now += 1;+ await self.offer(named, idle.lent);+ expect(downloads).toEqual(['0.31.0', '0.31.0']);+ expect(calls).toEqual(['prepare 0.31.0', 'prepare 0.31.0']);+ expect(idle.restarts()).toBe(0);+ });++ it('tries a release Switch names next at once, whatever failed before', async () => {+ const failing = recordingInstaller({ activate: 'broken' });+ const { self, downloads } = updater({ installer: failing.installer });+ const idle = host();+ await self.offer(named, idle.lent);+ await self.offer({ ...named, version: '0.32.0' }, idle.lent);+ expect(downloads).toEqual(['0.31.0', '0.32.0']);+ });++ it('reports a restart that came back on the old version rather than looping', async () => {+ const { installer } = recordingInstaller();+ const first = updater({ installer });+ await first.self.offer(named, host().lent);+ // The service started it again, still on 0.30.0, with the attempt kept in the store.+ const saved = first.attempt();+ const again = updater({+ installer,+ attempts: { last: () => saved, save: () => {} },+ now: () => first.clock.now + 60_000,+ });+ const idle = host();+ await again.self.offer(named, idle.lent);+ expect(again.downloads).toEqual([]);+ expect(again.self.status()).toMatchObject({ target: '0.31.0', state: 'failed' });+ expect(idle.restarts()).toBe(0);+ });++ it('says why when it cannot install a release itself', async () => {+ const { self, downloads } = updater({+ installer: unavailableInstaller('This controller runs from a checkout.'),+ });+ await self.offer(named, host().lent);+ expect(downloads).toEqual([]);+ expect(self.status()).toEqual({+ target: '0.31.0',+ state: 'failed',+ problem: 'This controller runs from a checkout.',+ });+ });++ it('outside a service, installs and says a restart is needed rather than exiting', async () => {+ const { installer, calls } = recordingInstaller();+ const { self } = updater({ installer, supervised: false });+ const idle = host();+ await self.offer(named, idle.lent);+ expect(calls).toEqual(['prepare 0.31.0', 'activate 0.31.0']);+ expect(idle.restarts()).toBe(0);+ expect(self.status()).toMatchObject({ target: '0.31.0', state: 'restart_needed' });+ await self.offer(named, idle.lent);+ expect(calls).toHaveLength(2);+ });++ it('stops short of switching when the controller is stopping', async () => {+ const { installer, calls } = recordingInstaller();+ const { self, attempt } = updater({ installer });+ const stopping = host();+ const offered = self.offer(named, stopping.lent);+ stopping.stop.abort();+ await offered;+ expect(calls).not.toContain('activate 0.31.0');+ expect(stopping.restarts()).toBe(0);+ expect(attempt()).toBeNull();+ });+});diff --git a/console/packages/agent-controller/src/self-update.ts b/console/packages/agent-controller/src/self-update.tsnew file mode 100644index 000000000..0b2c7e0e9--- /dev/null+++ b/console/packages/agent-controller/src/self-update.ts@@ -0,0 +1,476 @@+import { execFile } from 'node:child_process';+import { createHash, randomUUID } from 'node:crypto';+import { existsSync } from 'node:fs';+import { mkdir, open, rename, rm, stat } from 'node:fs/promises';+import { delimiter, dirname, join } from 'node:path';+import { z } from 'zod';+import type { Fetch } from './api';+import { errorMessage, type Logger } from './log';+import { pointCurrent, prefixBin, pruneReleases, readCurrent } from './releases';+import { RELEASE_VERSION } from './update';++/**+ * Updating this controller to the release Switch names in its status+ * responses (`controller_release`): fetched, checked against the SHA-256+ * Switch gave, installed, then switched to by exiting with `EXIT_RESTARTING`+ * for the service to start it again. Only while no session runs.+ */++/** The largest package a download is allowed to be. */+export const MAX_PACKAGE_BYTES = 200 * 1024 * 1024;+/** How long a failed attempt at one release keeps another from starting. */+export const RETRY_AFTER_MS = 60 * 60 * 1000;+const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000;+const INSTALL_TIMEOUT_MS = 10 * 60 * 1000;+const MAX_PROBLEM = 500;++/** The release Switch names for this controller. */+export type ControllerReleaseTarget = { version: string; url: string; sha256: string };++const controllerReleaseSchema = z.object({+ version: z.string().regex(RELEASE_VERSION, 'must be x.y.z or x.y.z-<prerelease>'),+ url: z.url({ protocol: /^https?$/ }),+ sha256: z+ .string()+ .regex(/^[0-9a-fA-F]{64}$/, 'must be 64 hex digits')+ .transform((value) => value.toLowerCase()),+});++/**+ * `controller_release` as a status response carries it: absent or null when+ * Switch manages no release for this controller. One that cannot be read is+ * treated as none, with the reason.+ */+export function readControllerRelease(raw: unknown): {+ release: ControllerReleaseTarget | null;+ problem: string | null;+} {+ if (raw === undefined || raw === null) return { release: null, problem: null };+ const parsed = controllerReleaseSchema.safeParse(raw);+ if (parsed.success) return { release: parsed.data, problem: null };+ return {+ release: null,+ problem: parsed.error.issues+ .map((issue) => `${issue.path.join('.') || 'controller_release'}: ${issue.message}`)+ .join('; '),+ };+}++/** Fetches `url` into a new file in `dir`, refusing more than `maxBytes`; the file and its SHA-256. */+export async function downloadPackage(input: {+ fetch: Fetch;+ url: string;+ dir: string;+ maxBytes: number;+ signal: AbortSignal | null;+}): Promise<{ file: string; sha256: string }> {+ await mkdir(input.dir, { recursive: true, mode: 0o700 });+ const signals = [+ AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),+ ...(input.signal ? [input.signal] : []),+ ];+ const response = await input.fetch(input.url, { signal: AbortSignal.any(signals) });+ if (!response.ok || !response.body)+ throw new Error(`Downloading ${input.url} answered ${response.status}.`);+ const declared = Number(response.headers.get('content-length') ?? Number.NaN);+ if (Number.isFinite(declared) && declared > input.maxBytes) {+ await response.body.cancel();+ throw new Error(+ `${input.url} is ${declared} bytes, more than the ${input.maxBytes} a controller package may be.`+ );+ }+ const file = join(input.dir, `switch-agent-controller-${randomUUID()}.tgz`);+ const handle = await open(file, 'wx', 0o600);+ const hash = createHash('sha256');+ let size = 0;+ try {+ for await (const chunk of response.body as unknown as AsyncIterable<Uint8Array>) {+ size += chunk.byteLength;+ if (size > input.maxBytes)+ throw new Error(+ `${input.url} sent more than the ${input.maxBytes} bytes a controller package may be.`+ );+ hash.update(chunk);+ await handle.write(chunk);+ }+ } catch (error) {+ await handle.close();+ await rm(file, { force: true });+ throw error;+ }+ await handle.close();+ return { file, sha256: hash.digest('hex') };+}++/**+ * Fetches the release into `dir` and checks it against the SHA-256 Switch+ * gave, the only one trusted: a file that differs is deleted.+ */+export async function downloadVerified(input: {+ fetch: Fetch;+ release: ControllerReleaseTarget;+ dir: string;+ maxBytes: number;+ signal: AbortSignal | null;+}): Promise<string> {+ const { file, sha256 } = await downloadPackage({ ...input, url: input.release.url });+ if (sha256 !== input.release.sha256) {+ await rm(file, { force: true });+ throw new Error(+ `The package at ${input.release.url} has SHA-256 ${sha256}, not the ${input.release.sha256} Switch named; it was deleted.`+ );+ }+ return file;+}++/** Runs a command to completion with its own environment; rejects with its stderr when it fails. */+export type InstallRunner = (+ file: string,+ args: string[],+ options: { env: NodeJS.ProcessEnv; signal: AbortSignal | null }+) => Promise<{ stdout: string }>;++export const runInstallCommand: InstallRunner = (file, args, options) =>+ new Promise((done, fail) => {+ execFile(+ file,+ args,+ {+ env: options.env,+ timeout: INSTALL_TIMEOUT_MS,+ maxBuffer: 16 * 1024 * 1024,+ ...(options.signal ? { signal: options.signal } : {}),+ },+ (error, stdout, stderr) => {+ if (error) {+ const said = stderr.trim().split('\n').slice(-5).join('\n');+ fail(new Error(`${file} ${args.join(' ')} failed: ${said || error.message}`));+ return;+ }+ done({ stdout });+ }+ );+ });++/** How npm is run to install a release: beside `node`, on a PATH that finds that `node` first. */+export type NpmSetup = {+ run: InstallRunner;+ node: string;+ /** npm's cache, in a directory the controller owns. */+ cacheDir: string;+ env: NodeJS.ProcessEnv;+};++function npmCommand(setup: NpmSetup): { file: string; env: NodeJS.ProcessEnv } {+ const beside = join(dirname(setup.node), 'npm');+ return {+ file: existsSync(beside) ? beside : 'npm',+ env: {+ ...setup.env,+ PATH: [dirname(setup.node), setup.env.PATH ?? ''].filter(Boolean).join(delimiter),+ npm_config_cache: setup.cacheDir,+ npm_config_update_notifier: 'false',+ npm_config_fund: 'false',+ npm_config_audit: 'false',+ // Agents running as users of their own read the shared-host bundle installed.+ npm_config_umask: '0022',+ },+ };+}++/**+ * `npm install --global --prefix <prefix> <file>`: the package from the local+ * file, never a URL. With no prefix, into npm's own global one.+ */+export async function npmInstall(+ setup: NpmSetup,+ prefix: string | null,+ file: string,+ signal: AbortSignal | null+): Promise<void> {+ const npm = npmCommand(setup);+ await setup.run(+ npm.file,+ ['install', '--global', ...(prefix ? ['--prefix', prefix] : []), file],+ {+ env: npm.env,+ signal,+ }+ );+}++/** Runs the CLI at `cli` with `--version`; it must say exactly `version`. */+export async function assertCliVersion(+ setup: NpmSetup,+ cli: string,+ version: string+): Promise<void> {+ const { stdout } = await setup.run(setup.node, [cli, '--version'], {+ env: npmCommand(setup).env,+ signal: null,+ });+ const said = stdout.trim();+ if (said !== version)+ throw new Error(`The installed controller says it is ${said || 'nothing'}, not ${version}.`);+}++/**+ * Installs a downloaded release. `prepare` readies it without changing what+ * runs; `activate` makes it what the next start runs.+ */+export type ReleaseInstaller = {+ /** Why this controller cannot install a release for itself; null when it can. */+ readonly unavailable: string | null;+ prepare(version: string, file: string, signal: AbortSignal | null): Promise<void>;+ activate(version: string, file: string): Promise<void>;+};++export function unavailableInstaller(reason: string): ReleaseInstaller {+ const refuse = async () => {+ throw new Error(reason);+ };+ return { unavailable: reason, prepare: refuse, activate: refuse };+}++/**+ * Installs into the npm prefix the running controller came from, over it, as+ * `update` does: a user's service, or a plain install.+ */+export class PrefixInstaller implements ReleaseInstaller {+ readonly unavailable = null;++ constructor(private readonly deps: { npm: NpmSetup; prefix: string; cli: string }) {}++ async prepare(): Promise<void> {}++ async activate(version: string, file: string): Promise<void> {+ await npmInstall(this.deps.npm, this.deps.prefix, file, null);+ await assertCliVersion(this.deps.npm, this.deps.cli, version);+ }+}++/**+ * Installs into a releases directory (`releases.ts`): into+ * `<version>.partial`, checked, renamed to `<version>`, then `current` is+ * pointed at it and older releases are pruned.+ */+export class ReleasesDirInstaller implements ReleaseInstaller {+ readonly unavailable = null;++ constructor(private readonly deps: { npm: NpmSetup; releasesDir: string }) {}++ async prepare(version: string, file: string, signal: AbortSignal | null): Promise<void> {+ const partial = join(this.deps.releasesDir, `${version}.partial`);+ const final = join(this.deps.releasesDir, version);+ await rm(partial, { recursive: true, force: true });+ try {+ await npmInstall(this.deps.npm, partial, file, signal);+ await assertCliVersion(this.deps.npm, prefixBin(partial), version);+ } catch (error) {+ await rm(partial, { recursive: true, force: true });+ throw error;+ }+ if (await exists(final)) {+ const replaced = `${final}.replaced-${randomUUID()}`;+ await rename(final, replaced);+ await rename(partial, final);+ await rm(replaced, { recursive: true, force: true });+ } else await rename(partial, final);+ }++ async activate(version: string): Promise<void> {+ const before = await readCurrent(this.deps.releasesDir);+ const target = join(this.deps.releasesDir, version);+ await pointCurrent(this.deps.releasesDir, target, null);+ await pruneReleases(this.deps.releasesDir, [+ target,+ ...(before.kind === 'link' ? [before.target] : []),+ ]);+ }+}++async function exists(path: string): Promise<boolean> {+ try {+ await stat(path);+ return true;+ } catch {+ return false;+ }+}++/** What the controller reports about an update it has not made (`update` in its status). */+export type UpdateReport = {+ target: string;+ state: 'waiting_idle' | 'failed' | 'restart_needed';+ problem?: string;+};++/** The last attempt at a release, kept across restarts so a failing one is not retried at each start. */+export type UpdateAttempt = { target: string; at: number; problem: string | null };++export type SelfUpdaterDeps = {+ version: string;+ installer: ReleaseInstaller;+ /** Fetches the release and checks it; the verified file, which the updater deletes. */+ download: (release: ControllerReleaseTarget, signal: AbortSignal) => Promise<string>;+ attempts: { last(): UpdateAttempt | null; save(attempt: UpdateAttempt | null): void };+ /** Whether exiting with `EXIT_RESTARTING` brings the controller back, as a service does. */+ supervised: boolean;+ log: Logger;+ now: () => number;+};++/** What the running controller lends the updater for one offer. */+export type UpdateHost = {+ /** Whether no session runs here now. */+ idle: () => Promise<boolean>;+ /** Ends the controller with `restarting`. */+ restart: () => void;+ /** The controller stopping. */+ signal: AbortSignal;+};++/**+ * Follows the release Switch names: a version other than this one (newer or+ * older, so Switch can roll back) is installed while no session runs, and+ * the controller restarts into it. A failed attempt is not repeated for an+ * hour; what stands in the way is reported (`status()`).+ */+export class SelfUpdater {+ private report: UpdateReport | null = null;+ private running: Promise<void> | null = null;+ private lastLogged: string | null = null;++ constructor(private readonly deps: SelfUpdaterDeps) {}++ status(): UpdateReport | null {+ return this.report;+ }++ /** Considers what a status response named; one offer at a time, the others are dropped. */+ offer(raw: unknown, host: UpdateHost): Promise<void> {+ if (this.running) return this.running;+ this.running = this.consider(raw, host)+ .catch((error: unknown) => {+ this.deps.log.error('Following the release Switch names failed', {+ error: errorMessage(error),+ });+ })+ .finally(() => {+ this.running = null;+ });+ return this.running;+ }++ /** Resolves once no offer is being worked on. */+ async settled(): Promise<void> {+ await this.running;+ }++ private async consider(raw: unknown, host: UpdateHost): Promise<void> {+ const { log, version, installer } = this.deps;+ const { release, problem } = readControllerRelease(raw);+ if (problem)+ this.logOnce(`malformed:${problem}`, () =>+ log.warn('Switch named a controller release this controller cannot read; ignoring it', {+ problem,+ })+ );+ if (!release || release.version === version) {+ this.report = null;+ this.lastLogged = null;+ return;+ }+ const target = release.version;+ if (this.report?.target === target && this.report.state === 'restart_needed') return;+ if (installer.unavailable) {+ this.fail(target, installer.unavailable, false);+ return;+ }+ const last = this.deps.attempts.last();+ if (last && last.target === target && this.deps.now() - last.at < RETRY_AFTER_MS) {+ this.fail(+ target,+ last.problem ??+ `An attempt at ${target} at ${new Date(last.at).toISOString()} did not finish, or restarted into it and still ran ${version}.`,+ true+ );+ return;+ }+ if (!(await host.idle())) return this.waitForIdle(target);+ if (host.signal.aborted) return;+ log.info('Updating this controller to the release Switch names', { from: version, to: target });+ const at = this.deps.now();+ this.deps.attempts.save({ target, at, problem: null });+ let file: string | null = null;+ try {+ file = await this.deps.download(release, host.signal);+ await installer.prepare(target, file, host.signal);+ if (host.signal.aborted) {+ this.deps.attempts.save(null);+ return;+ }+ if (!(await host.idle())) {+ this.deps.attempts.save(null);+ return this.waitForIdle(target);+ }+ await installer.activate(target, file);+ } catch (error) {+ if (host.signal.aborted) {+ this.deps.attempts.save(null);+ return;+ }+ const problem = errorMessage(error).slice(0, MAX_PROBLEM);+ this.deps.attempts.save({ target, at, problem });+ this.fail(target, problem, true);+ return;+ } finally {+ if (file) await rm(file, { force: true });+ }+ if (this.deps.supervised) {+ log.info('Installed the release Switch names; restarting into it', { version: target });+ this.report = null;+ host.restart();+ return;+ }+ this.report = {+ target,+ state: 'restart_needed',+ problem: `Installed ${target}; this controller runs outside a service, so it runs ${version} until it is restarted.`,+ };+ log.warn(+ `Installed switch-agent-controller ${target}, the release Switch names; restart this controller to run it (it runs ${version}).`+ );+ }++ private waitForIdle(target: string): void {+ this.report = { target, state: 'waiting_idle' };+ this.logOnce(`waiting:${target}`, () =>+ this.deps.log.info(+ 'Switch names another controller release; it is installed once no session runs',+ {+ from: this.deps.version,+ to: target,+ }+ )+ );+ }++ private fail(target: string, problem: string, retrying: boolean): void {+ this.report = { target, state: 'failed', problem: problem.slice(0, MAX_PROBLEM) };+ this.logOnce(`failed:${target}:${problem}`, () =>+ this.deps.log.warn(+ retrying+ ? 'This controller could not update to the release Switch names; it tries again in an hour'+ : 'This controller cannot update itself to the release Switch names',+ { from: this.deps.version, to: target, problem }+ )+ );+ }++ private logOnce(key: string, write: () => void): void {+ if (this.lastLogged === key) return;+ this.lastLogged = key;+ write();+ }+}diff --git a/console/packages/agent-controller/src/separate-users.test.ts b/console/packages/agent-controller/src/separate-users.test.tsindex b8068fb24..6bd26207e 100644--- a/console/packages/agent-controller/src/separate-users.test.ts+++ b/console/packages/agent-controller/src/separate-users.test.ts@@ -1,9 +1,24 @@-import { describe, expect, it } from 'vitest';+import {+ mkdirSync,+ mkdtempSync,+ readlinkSync,+ rmSync,+ statSync,+ symlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, it } from 'vitest';+import { packageDir } from './releases';import {agentPath,agentUnitTemplate,controllerSystemUnit,+ defaultReleasesDir,+ ensureReleases,homeRootOf,+ planReleases,parsePasswd,polkitRule,type SeparateUsersConfig,@@ -23,6 +38,7 @@ const config: SeparateUsersConfig = {agentUsers: 4,node: '/usr/bin/node',bundle: '/usr/lib/node_modules/switch-agent-controller/shared-host.mjs',+ releasesDir: null,};function lines(text: string): string[] {@@ -132,3 +148,160 @@ describe.skipIf(process.platform === 'win32')('separate users', () => {expect(separateUsersConfigSchema.safeParse({ ...config, agentUsers: 100 }).success).toBe(false);});});++describe.skipIf(process.platform === 'win32')('releases directory', () => {+ const baked = {+ cli: '/opt/switch/controller/lib/node_modules/switch-agent-controller/cli.mjs',+ bundle: '/opt/switch/controller/lib/node_modules/switch-agent-controller/shared-host.mjs',+ uid: 2000,+ dataDir: '/data/.switch-controller',+ agentsDir: '/data/agents',+ homes: ['/nonexistent'],+ };++ it('runs the controller and its agents through current', () => {+ const plan = planReleases({ ...baked, releasesDir: '/data/controller-releases' });+ expect(plan).toEqual({+ dir: '/data/controller-releases',+ prefix: '/opt/switch/controller',+ cli: '/data/controller-releases/current/lib/node_modules/switch-agent-controller/cli.mjs',+ bundle:+ '/data/controller-releases/current/lib/node_modules/switch-agent-controller/shared-host.mjs',+ });+ expect(planReleases({ ...baked, releasesDir: null })?.dir).toBe(+ '/var/lib/switch-agent-controller-2000/releases'+ );+ expect(defaultReleasesDir(2000)).toBe('/var/lib/switch-agent-controller-2000/releases');++ const unitConfig: SeparateUsersConfig = {+ ...config,+ dataDir: baked.dataDir,+ agentsDir: baked.agentsDir,+ bundle: plan!.bundle,+ releasesDir: plan!.dir,+ };+ const agentUnit = lines(+ agentUnitTemplate(unitConfig, {+ sh: '/bin/sh',+ find: '/usr/bin/find',+ chown: '/usr/bin/chown',+ path: '/usr/bin:/bin',+ })+ );+ expect(agentUnit).toContain(+ 'ExecStart=/usr/bin/node /data/controller-releases/current/lib/node_modules/switch-agent-controller/shared-host.mjs /data/agents/agent/watcher /data/agents/agent/watcher/config.json --watch-worker'+ );+ const controllerUnit = lines(+ controllerSystemUnit(unitConfig, { cli: plan!.cli, envFile: null, path: '/usr/bin:/bin' })+ );+ const exec = controllerUnit.find((line) => line.startsWith('ExecStart='))!;+ expect(exec).toContain(+ '"/usr/bin/node" "/data/controller-releases/current/lib/node_modules/switch-agent-controller/cli.mjs" "run"'+ );+ expect(exec).toContain(+ '"--shared-host-bundle" "/data/controller-releases/current/lib/node_modules/switch-agent-controller/shared-host.mjs"'+ );+ // 7 is restarting into an update: systemd must start it again.+ expect(controllerUnit).toContain('Restart=on-failure');+ const prevented = controllerUnit.find((line) => line.startsWith('RestartPreventExitStatus='))!;+ expect(prevented.split('=')[1]!.split(' ')).not.toContain('7');+ });++ it('keeps a bundle named outside the controller’s install where it is', () => {+ expect(+ planReleases({ ...baked, bundle: '/opt/shared-host.mjs', releasesDir: null })?.bundle+ ).toBe('/opt/shared-host.mjs');+ });++ it('has none for a controller that is not an npm install, and refuses one named for it', () => {+ const checkout = {+ ...baked,+ cli: '/srv/switch/console/packages/agent-controller/dist/cli.mjs',+ };+ expect(planReleases({ ...checkout, releasesDir: null })).toBeNull();+ expect(() => planReleases({ ...checkout, releasesDir: '/srv/releases' })).toThrow(+ /--releases-dir needs the controller installed with npm/+ );+ });++ it('refuses a directory agents cannot read', () => {+ for (const releasesDir of [+ '/data/.switch-controller/releases',+ '/data/agents/releases',+ '/home/builder/releases',+ '/nonexistent/releases',+ '/data',+ '/srv/with space',+ ])+ expect(() => planReleases({ ...baked, releasesDir }), releasesDir).toThrow(/--releases-dir/);+ });+});++describe.skipIf(process.platform === 'win32')('ensureReleases', () => {+ let dir: string;+ let releases: string;++ beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), 'ensure-releases-'));+ releases = join(dir, 'var', 'releases');+ });++ afterEach(() => {+ rmSync(dir, { recursive: true, force: true });+ });++ function release(prefix: string, version: string): string {+ mkdirSync(packageDir(prefix), { recursive: true });+ writeFileSync(join(packageDir(prefix), 'package.json'), JSON.stringify({ version }));+ return prefix;+ }++ const ensure = (prefix: string, version: string) =>+ ensureReleases({ releasesDir: releases, prefix, version, owner: null });++ it('makes the directory, readable by all, and points current at the installing release', async () => {+ const baked = release(join(dir, 'opt'), '0.30.0');+ await ensure(baked, '0.30.0');+ expect(statSync(releases).mode & 0o777).toBe(0o755);+ expect(readlinkSync(join(releases, 'current'))).toBe(baked);+ });++ it('replaces a dangling current, and one older than the installing release', async () => {+ const baked = release(join(dir, 'opt'), '0.30.0');+ mkdirSync(releases, { recursive: true });+ symlinkSync(join(releases, '0.29.0'), join(releases, 'current'));+ await ensure(baked, '0.30.0');+ expect(readlinkSync(join(releases, 'current'))).toBe(baked);++ rmSync(join(releases, 'current'));+ symlinkSync(release(join(releases, '0.29.0'), '0.29.0'), join(releases, 'current'));+ await ensure(baked, '0.30.0');+ expect(readlinkSync(join(releases, 'current'))).toBe(baked);+ });++ it('keeps a current as new as the installing release, or newer: an update applied since', async () => {+ const baked = release(join(dir, 'opt'), '0.30.0');+ mkdirSync(releases, { recursive: true });+ const updated = release(join(releases, '0.31.0'), '0.31.0');+ symlinkSync(updated, join(releases, 'current'));+ const note = await ensure(baked, '0.30.0');+ expect(readlinkSync(join(releases, 'current'))).toBe(updated);+ expect(note).toContain('0.31.0');+ // A newer image wins.+ const newer = release(join(dir, 'opt-new'), '0.32.0');+ await ensure(newer, '0.32.0');+ expect(readlinkSync(join(releases, 'current'))).toBe(newer);+ await ensure(newer, '0.32.0');+ expect(readlinkSync(join(releases, 'current'))).toBe(newer);+ });++ it('refuses a current that is not a link, and a directory that is a link', async () => {+ const baked = release(join(dir, 'opt'), '0.30.0');+ mkdirSync(join(releases, 'current'), { recursive: true });+ await expect(ensure(baked, '0.30.0')).rejects.toThrow(/is not a link/);+ rmSync(releases, { recursive: true });+ mkdirSync(join(dir, 'elsewhere'));+ symlinkSync(join(dir, 'elsewhere'), releases);+ await expect(ensure(baked, '0.30.0')).rejects.toThrow(/is not a directory/);+ });+});diff --git a/console/packages/agent-controller/src/separate-users.ts b/console/packages/agent-controller/src/separate-users.tsindex fea21f7a9..c003af044 100644--- a/console/packages/agent-controller/src/separate-users.ts+++ b/console/packages/agent-controller/src/separate-users.ts@@ -2,6 +2,7 @@ import {access,chmod,chown,+ lstat,mkdir,readdir,readFile,@@ -13,7 +14,9 @@ import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from 'noimport { z } from 'zod';import { ConfigurationError } from './errors';import { defaultDataDir } from './paths';+import { CURRENT, pointCurrent, readCurrent } from './releases';import { CommandFailure, type CommandRunner } from './secrets';+import { installPrefix, isNewer } from './update';/*** Running every agent as a Linux user of its own.@@ -36,6 +39,7 @@ export const SEPARATE_USERS_CONFIG_DIR = '/etc/switch-agent-controller';const SYSTEMD_SYSTEM_DIR = '/etc/systemd/system';const POLKIT_RULES_DIR = '/etc/polkit-1/rules.d';const AGENTS_PARENT = '/var/lib/switch-agents';+const RELEASES_PARENT = '/var/lib';export const MAX_AGENT_USERS = 99;export const DEFAULT_AGENT_USERS = 16;/** Where agents cannot reach, so nothing an agent runs may live there. */@@ -55,6 +59,12 @@ export const separateUsersConfigSchema = z.strictObject({agentUsers: z.number().int().min(1).max(MAX_AGENT_USERS),node: absolutePath,bundle: absolutePath,+ /**+ * The releases directory the controller installs the releases Switch names+ * into (`releases.ts`); null for a setup made before there was one, or from+ * a controller that is not an npm install.+ */+ releasesDir: absolutePath.nullable().default(null),});export type SeparateUsersConfig = z.infer<typeof separateUsersConfigSchema>;@@ -88,6 +98,11 @@ export function defaultAgentsDir(uid: number): string {return join(AGENTS_PARENT, String(uid));}+/** Where the controller's releases are when setup names no other place. */+export function defaultReleasesDir(uid: number): string {+ return join(RELEASES_PARENT, `switch-agent-controller-${uid}`, 'releases');+}+/** Where the directory of the agent agent user `slot` runs is, on the machine. */export function agentRoot(config: SeparateUsersConfig, slot: number): string {return join(config.agentsDir, slotName(slot));@@ -225,7 +240,7 @@ polkit.addRule(function (action, subject) {/*** The controller as a system service of its user, with the agents' group as a* supplementary group, so it can read what its agents write. Restarted on exit- * code 1 only, as the user service is.+ * codes 1 and 7 (restarting into an update), as the user service is.*/export function controllerSystemUnit(config: SeparateUsersConfig,@@ -356,6 +371,10 @@ export type SetupInput = {node: string;cli: string;bundle: string;+ /** The version of the controller running the setup, which `cli` is. */+ version: string;+ /** Where the controller's releases go; `defaultReleasesDir` when null. */+ releasesDir: string | null;envFile: string | null;path: string;/**@@ -412,6 +431,15 @@ export async function installSeparateUsers(throw new ConfigurationError(`--agents-dir ${agentsDir} is under a home directory, which agents cannot reach.`);+ const releases = planReleases({+ cli: input.cli,+ bundle: input.bundle,+ releasesDir: input.releasesDir,+ uid: account.uid,+ dataDir,+ agentsDir: resolve(agentsDir),+ homes,+ });const config = separateUsersConfigSchema.parse({v: 1,user: account.name,@@ -421,7 +449,8 @@ export async function installSeparateUsers(agentsDir: resolve(agentsDir),agentUsers: input.agentUsers,node: input.node,- bundle: input.bundle,+ bundle: releases?.bundle ?? input.bundle,+ releasesDir: releases?.dir ?? null,} satisfies SeparateUsersConfig);for (let slot = 1; slot <= config.agentUsers; slot++)await ensureAgentUser(names.agentUser(slot), names.group, run);@@ -430,6 +459,14 @@ export async function installSeparateUsers(await mkdir(config.agentsDir, { recursive: true, mode: 0o700 });await chown(config.agentsDir, account.uid, account.gid);await chmod(config.agentsDir, 0o700);+ const releasesNote = releases+ ? await ensureReleases({+ releasesDir: releases.dir,+ prefix: releases.prefix,+ version: input.version,+ owner: { uid: account.uid, gid: account.gid },+ })+ : 'The controller does not run from an npm install, so it cannot install the releases Switch names for itself.';const tools = {sh: await which('sh', SYSTEM_PATH),@@ -445,7 +482,11 @@ export async function installSeparateUsers(await writeFile(names.polkitRule, polkitRule(config), { mode: 0o644 });await writeFile(join(SYSTEMD_SYSTEM_DIR, names.controllerUnit),- controllerSystemUnit(config, { cli: input.cli, envFile: input.envFile, path: input.path }),+ controllerSystemUnit(config, {+ cli: releases?.cli ?? input.cli,+ envFile: input.envFile,+ path: input.path,+ }),{ mode: 0o644 });await run('systemctl', ['daemon-reload'], null);@@ -463,12 +504,116 @@ export async function installSeparateUsers(`Agents' directories: ${config.agentsDir}`,`Each agent runs as ${names.agentUnitPattern}; its log: journalctl -u ${names.agentUnit(1)}`,`The controller runs as ${names.controllerUnit}; its log: journalctl -u ${names.controllerUnit} -f`,+ releasesNote,'Agents see only their own directory: a definition naming a directory elsewhere is refused.',"Agents do not see this user's provider logins: give each provider an API key or token in --env-file.",],};}+/** Where the controller's units run it and its bundle from, through `<releases>/current`. */+export type ReleasesPlan = {+ dir: string;+ /** The npm prefix the controller running the setup was installed in. */+ prefix: string;+ cli: string;+ bundle: string;+};++/**+ * The releases directory for a setup run by the controller at `cli`, and the+ * paths of the CLI and the shared-host bundle through its `current` link; null+ * when that controller is not an npm install (a checkout), which cannot have+ * one. The directory must be where agents can read it and the controller+ * alone writes: not under a home, the data directory or the agents'+ * directories, which agents cannot reach.+ */+export function planReleases(input: {+ cli: string;+ bundle: string;+ releasesDir: string | null;+ uid: number;+ dataDir: string;+ agentsDir: string;+ homes: string[];+}): ReleasesPlan | null {+ const prefix = installPrefix(input.cli);+ if (!prefix) {+ if (input.releasesDir)+ throw new ConfigurationError(+ `--releases-dir needs the controller installed with npm (sudo npm install --global <the release's .tgz URL>), not run from ${input.cli}.`+ );+ return null;+ }+ const dir = resolve(input.releasesDir ?? defaultReleasesDir(input.uid));+ if (!isUnitSafe(dir))+ throw new ConfigurationError(+ `--releases-dir ${dir}: a path in a unit file must hold only letters, digits and ._/@+-.`+ );+ const home = homeRootOf(dir, input.homes);+ if (home)+ throw new ConfigurationError(+ `--releases-dir ${dir} is under ${home}, which agents running as users of their own cannot reach.`+ );+ for (const [what, root] of [+ ['the data directory', input.dataDir],+ ["the agents' directories", input.agentsDir],+ ] as const)+ if (dir === resolve(root) || isWithin(root, dir) || isWithin(dir, root))+ throw new ConfigurationError(+ `--releases-dir ${dir} overlaps ${what} (${root}), which agents cannot read; name a directory of its own.`+ );+ const current = join(dir, CURRENT);+ return {+ dir,+ prefix,+ cli: join(current, relative(prefix, input.cli)),+ bundle: isWithin(prefix, input.bundle)+ ? join(current, relative(prefix, input.bundle))+ : input.bundle,+ };+}++/**+ * Makes the releases directory (0755, the controller's), and points its+ * `current` link at `prefix`, the release running the setup, only when the+ * link is missing, names no readable release, or names one older than+ * `version`: a machine whose boot runs the setup with the release its image+ * carries keeps an update applied since, and a newer image still wins.+ */+export async function ensureReleases(input: {+ releasesDir: string;+ prefix: string;+ version: string;+ owner: { uid: number; gid: number } | null;+}): Promise<string> {+ const { releasesDir } = input;+ await mkdir(dirname(releasesDir), { recursive: true, mode: 0o755 });+ try {+ const found = await lstat(releasesDir);+ if (!found.isDirectory() || found.isSymbolicLink())+ throw new ConfigurationError(`${releasesDir} is not a directory; move it aside.`);+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;+ await mkdir(releasesDir, { mode: 0o755 });+ }+ if (input.owner) await chown(releasesDir, input.owner.uid, input.owner.gid);+ await chmod(releasesDir, 0o755);+ const current = await readCurrent(releasesDir);+ if (current.kind === 'not_a_link')+ throw new ConfigurationError(+ `${join(releasesDir, CURRENT)} is not a link; move it aside and run the setup again.`+ );+ if (+ current.kind === 'link' &&+ current.version !== null &&+ !isNewer(input.version, current.version)+ )+ return `The controller runs ${current.version} from ${current.target}, through ${join(releasesDir, CURRENT)}; it installs the releases Switch names there.`;+ await pointCurrent(releasesDir, input.prefix, input.owner);+ return `The controller runs ${input.version} from ${input.prefix}, through ${join(releasesDir, CURRENT)}; it installs the releases Switch names there.`;+}+/*** Stops the controller and every agent unit, and removes the units, the rule,* the setup, the agent users and their group. The agents' directories are@@ -483,12 +628,14 @@ export async function uninstallSeparateUsers(const account = parsePasswd((await run('getent', ['passwd', user], null)).stdout);const names = separateUserNames(account.uid);let agentsDir: string | null = null;+ let releasesDir: string | null = null;let agentUsers = MAX_AGENT_USERS;try {const config = separateUsersConfigSchema.parse(JSON.parse(await readFile(names.config, 'utf8')));agentsDir = config.agentsDir;+ releasesDir = config.releasesDir;agentUsers = config.agentUsers;} catch (error) {if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;@@ -505,11 +652,16 @@ export async function uninstallSeparateUsers(await run('userdel', [names.agentUser(slot)], null).catch(ignoreFailure);await run('groupdel', [names.group], null).catch(ignoreFailure);return {- notes: agentsDir- ? [- `The agents' directories are kept in ${agentsDir}; remove them when nothing in them is needed.`,- ]- : [],+ notes: [+ ...(agentsDir+ ? [+ `The agents' directories are kept in ${agentsDir}; remove them when nothing in them is needed.`,+ ]+ : []),+ ...(releasesDir+ ? [`The controller's releases are kept in ${releasesDir}; remove it when it is not needed.`]+ : []),+ ],};}diff --git a/console/packages/agent-controller/src/service.test.ts b/console/packages/agent-controller/src/service.test.tsindex a6f067082..ce14e5074 100644--- a/console/packages/agent-controller/src/service.test.ts+++ b/console/packages/agent-controller/src/service.test.ts@@ -1,7 +1,14 @@import { homedir } from 'node:os';import { describe, expect, it } from 'vitest';+import { EXIT_RESTARTING } from './exit-codes';import { defaultDataDir } from './paths';-import { launchdPlist, serviceIdentity, systemdUnit, type ServiceSpec } from './service';+import {+ LAUNCHD_FINAL_EXIT_CODES,+ launchdPlist,+ serviceIdentity,+ systemdUnit,+ type ServiceSpec,+} from './service';const SPEC: ServiceSpec = {node: '/usr/bin/node',@@ -9,6 +16,7 @@ const SPEC: ServiceSpec = {dataDir: '/srv/controller data',envFile: '/home/me/agents.env',sharedHostBundle: null,+ releasesDir: null,path: '/home/me/.local/bin:/usr/bin:/bin',};@@ -23,6 +31,24 @@ describe('systemdUnit', () => {expect(unit).toContain('RestartPreventExitStatus=2 3 4 5 6');expect(unit).not.toContain('--launchd');});++ it('runs through the releases directory, and tells the controller where it is', () => {+ const unit = systemdUnit({+ ...SPEC,+ cli: '/srv/controller data/releases/current/lib/node_modules/switch-agent-controller/cli.mjs',+ releasesDir: '/srv/controller data/releases',+ });+ expect(unit).toContain(+ '"/srv/controller data/releases/current/lib/node_modules/switch-agent-controller/cli.mjs" "run" "--data-dir" "/srv/controller data" "--env-file" "/home/me/agents.env" "--releases-dir" "/srv/controller data/releases"'+ );+ });++ it('restarts on 7, restarting into an update', () => {+ const prevented = systemdUnit(SPEC)+ .split('\n')+ .find((line) => line.startsWith('RestartPreventExitStatus='))!;+ expect(prevented.split('=')[1]!.split(' ')).not.toContain(String(EXIT_RESTARTING));+ });});describe('launchdPlist', () => {@@ -35,6 +61,8 @@ describe('launchdPlist', () => {expect(plist).toContain('<string>/Users/me/A & B</string>');expect(plist).toContain('<string>--launchd</string>');expect(plist).toMatch(/<key>SuccessfulExit<\/key>\s*<false\/>/);+ // 7 stays 7 under --launchd, so KeepAlive starts it again on its new release.+ expect(LAUNCHD_FINAL_EXIT_CODES).not.toContain(EXIT_RESTARTING);expect(plist).toContain('<key>StandardErrorPath</key>\n <string>/Users/me/A & B/controller.log</string>');diff --git a/console/packages/agent-controller/src/service.ts b/console/packages/agent-controller/src/service.tsindex a691d7a09..846d47783 100644--- a/console/packages/agent-controller/src/service.ts+++ b/console/packages/agent-controller/src/service.ts@@ -17,6 +17,8 @@ export type ServiceSpec = {dataDir: string;envFile: string | null;sharedHostBundle: string | null;+ /** The releases directory the service runs through (`<releases>/current`), and the controller installs releases into; null when it runs from where it is. */+ releasesDir: string | null;path: string;};@@ -54,13 +56,14 @@ function runArguments(spec: ServiceSpec, launchd: boolean): string[] {spec.dataDir,...(spec.envFile ? ['--env-file', spec.envFile] : []),...(spec.sharedHostBundle ? ['--shared-host-bundle', spec.sharedHostBundle] : []),+ ...(spec.releasesDir ? ['--releases-dir', spec.releasesDir] : []),...(launchd ? ['--launchd'] : []),];}/**- * The systemd user unit: restarted on exit code 1 only, since 2 (configuration),- * 3 (revoked) and 4 (taken over) would fail the same way again.+ * The systemd user unit: restarted on exit code 1 (may pass) and 7 (restarting+ * into an update), not on 2 to 6, which would fail the same way again.*/export function systemdUnit(spec: ServiceSpec): string {const quote = (value: string) => `"${value.replace(/(["\\])/g, '\\$1')}"`;@@ -95,7 +98,7 @@ function xml(value: string): string {* The launchd agent. launchd cannot spare particular exit codes from a* restart, so the controller runs with `--launchd`, which ends with 0 on the* codes that must not restart (having logged why), and `KeepAlive` restarts- * only an unsuccessful exit.+ * only an unsuccessful exit: 1, and 7 (restarting into an update).*/export function launchdPlist(spec: ServiceSpec, label: string, logFile: string): string {const strings = (values: string[]) =>diff --git a/console/packages/agent-controller/src/status.ts b/console/packages/agent-controller/src/status.tsindex fbb2ea233..7fa169897 100644--- a/console/packages/agent-controller/src/status.ts+++ b/console/packages/agent-controller/src/status.ts@@ -621,5 +621,8 @@ export function statusFingerprint(report: Omit<StatusReport, 'seq'>): string {a.sessions.ids,a.directory,]),+ update: report.update+ ? [report.update.target, report.update.state, report.update.problem]+ : null,});}diff --git a/console/packages/agent-controller/src/store.test.ts b/console/packages/agent-controller/src/store.test.tsindex 0a3323496..b60be1751 100644--- a/console/packages/agent-controller/src/store.test.ts+++ b/console/packages/agent-controller/src/store.test.ts@@ -42,6 +42,24 @@ const assignment = {};describe('ControllerStore', () => {+ it('keeps the last update attempt across reopening, and forgets it', () => {+ const store = ControllerStore.open(path);+ expect(store.updateAttempt()).toBeNull();+ store.saveUpdateAttempt({ target: '0.31.0', at: 42, problem: null });+ store.close();+ const again = ControllerStore.open(path);+ expect(again.updateAttempt()).toEqual({ target: '0.31.0', at: 42, problem: null });+ again.saveUpdateAttempt({ target: '0.31.0', at: 43, problem: 'npm install failed' });+ expect(again.updateAttempt()).toEqual({+ target: '0.31.0',+ at: 43,+ problem: 'npm install failed',+ });+ again.saveUpdateAttempt(null);+ expect(again.updateAttempt()).toBeNull();+ again.close();+ });+it('migrates a new file to the current schema version', () => {const store = ControllerStore.open(path);expect(store.schemaVersion()).toBe(STORE_SCHEMA_VERSION);diff --git a/console/packages/agent-controller/src/store.ts b/console/packages/agent-controller/src/store.tsindex 9612aaecf..e2dbc20d0 100644--- a/console/packages/agent-controller/src/store.ts+++ b/console/packages/agent-controller/src/store.ts@@ -3,6 +3,7 @@ import type { DatabaseSync } from 'node:sqlite';import { ConfigurationError } from './errors';import { errorMessage } from './log';import { type Assignment, assignmentSchema, type ReasonCode } from './schemas';+import type { UpdateAttempt } from './self-update';/*** The controller's local state, in one SQLite file: who it is, the last@@ -358,6 +359,29 @@ export class ControllerStore {});}+ /** The last attempt at updating to a release Switch named; see `SelfUpdater`. */+ updateAttempt(): UpdateAttempt | null {+ const value = this.meta('update_attempt');+ if (value === null) return null;+ try {+ const parsed = JSON.parse(value) as Partial<UpdateAttempt>;+ if (typeof parsed.target === 'string' && typeof parsed.at === 'number')+ return {+ target: parsed.target,+ at: parsed.at,+ problem: typeof parsed.problem === 'string' ? parsed.problem : null,+ };+ } catch {+ // Unreadable: as if no attempt was recorded.+ }+ return null;+ }++ saveUpdateAttempt(attempt: UpdateAttempt | null): void {+ if (attempt === null) this.db.prepare("DELETE FROM meta WHERE key = 'update_attempt'").run();+ else this.setMeta('update_attempt', JSON.stringify(attempt));+ }+private meta(key: string): string | null {const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as Row | undefined;return row ? String(row.value) : null;diff --git a/console/packages/agent-controller/src/systemd-runtime.e2e.test.ts b/console/packages/agent-controller/src/systemd-runtime.e2e.test.tsindex 25e8653e6..50b6f0e6a 100644--- a/console/packages/agent-controller/src/systemd-runtime.e2e.test.ts+++ b/console/packages/agent-controller/src/systemd-runtime.e2e.test.ts@@ -140,6 +140,7 @@ describe.skipIf(!dataDir)('agents as users of their own, under systemd', () => {dataDir: dataDir!,workspacesFor: () => join(dataDir!, 'workspaces'),version: '0.1.0',+ selfUpdate: null,now: Date.now,random: () => 0,timing: {diff --git a/console/packages/agent-controller/src/systemd-runtime.test.ts b/console/packages/agent-controller/src/systemd-runtime.test.tsindex 59d58bfd6..f2a86355d 100644--- a/console/packages/agent-controller/src/systemd-runtime.test.ts+++ b/console/packages/agent-controller/src/systemd-runtime.test.ts@@ -110,6 +110,7 @@ beforeEach(() => {agentUsers: 2,node: '/usr/bin/node',bundle: '/opt/switch/shared-host.mjs',+ releasesDir: null,};mkdirSync(config.dataDir, { mode: 0o700 });mkdirSync(config.agentsDir, { mode: 0o700 });diff --git a/console/packages/agent-controller/src/testing/fake-core.ts b/console/packages/agent-controller/src/testing/fake-core.tsindex 2549cc0f5..ea4c8078b 100644--- a/console/packages/agent-controller/src/testing/fake-core.ts+++ b/console/packages/agent-controller/src/testing/fake-core.ts@@ -108,6 +108,8 @@ export class FakeCore {publicKey: string | null = null;/** The logins given to the machine, by provider, as `GET .../provider-credentials/{provider}` answers. */readonly sealedLogins = new Map<string, { revision: number; sealed: unknown }>();+ /** What status responses name as `controller_release`; left out while undefined. */+ controllerRelease: unknown = undefined;/** Answers the next request to a path with this, once. */readonly scripted: { method: string; path: string; status: number; body: unknown }[] = [];/** What `GET .../media` sends, in these chunks; `waitBetween` holds back all but the first. */@@ -441,6 +443,9 @@ export class FakeCore {return this.json(res, 200, {assignment_revision: this.assignment.revision,report_within_s: this.reportWithinS,+ ...(this.controllerRelease === undefined+ ? {}+ : { controller_release: this.controllerRelease }),});}if (method === 'GET' && url.pathname === `${base}/operations`) {diff --git a/console/packages/agent-controller/src/update.test.ts b/console/packages/agent-controller/src/update.test.tsindex 85781aa68..d13d31a68 100644--- a/console/packages/agent-controller/src/update.test.ts+++ b/console/packages/agent-controller/src/update.test.ts@@ -25,7 +25,20 @@ describe('isNewer', () => {expect(isNewer('0.10.0', '0.9.9')).toBe(true);expect(isNewer('1.0.0', '1.0.0')).toBe(false);expect(isNewer('0.9.0', '0.10.0')).toBe(false);- expect(isNewer('1.0.0-rc.1', '0.1.0')).toBe(false);+ expect(isNewer('1.0', '0.1.0')).toBe(false);+ expect(isNewer('1.0.0+build.1', '0.1.0')).toBe(false);+ expect(isNewer('0.1.0', 'main')).toBe(false);+ });++ it('ranks a dev build after the release before it and before its own base', () => {+ expect(isNewer('0.31.1-dev.412.gabc1234', '0.31.0')).toBe(true);+ expect(isNewer('0.31.1', '0.31.1-dev.412.gabc1234')).toBe(true);+ expect(isNewer('0.31.1-dev.412.gabc1234', '0.31.1')).toBe(false);+ expect(isNewer('0.31.1-dev.12.g0000000', '0.31.1-dev.3.gfffffff')).toBe(true);+ expect(isNewer('0.31.1-dev.3.gfffffff', '0.31.1-dev.12.g0000000')).toBe(false);+ expect(isNewer('0.31.1-rc.1', '0.31.1-dev.99.g1')).toBe(true);+ expect(isNewer('1.0.0-alpha.1', '1.0.0-alpha')).toBe(true);+ expect(isNewer('1.0.0-alpha', '1.0.0-alpha')).toBe(false);});});@@ -36,6 +49,7 @@ describe('latestRelease', () => {release('switch-agent-controller-v0.2.0'),release('switch-agent-controller-v0.10.0'),release('switch-agent-controller-v0.11.0', { prerelease: true }),+ release('switch-agent-controller-v0.14.0-dev.3.gabc1234'),release('switch-agent-controller-v0.12.0', { draft: true }),{ ...release('switch-v9.9.9'), tag_name: 'switch-v9.9.9' },{ ...release('switch-agent-controller-v0.13.0'), assets: [] },diff --git a/console/packages/agent-controller/src/update.ts b/console/packages/agent-controller/src/update.tsindex 703877443..e6f451e84 100644--- a/console/packages/agent-controller/src/update.ts+++ b/console/packages/agent-controller/src/update.ts@@ -41,20 +41,56 @@ const releaseSchema = z.object({export type ControllerRelease = { version: string; packageUrl: string };-type Version = [number, number, number];+/**+ * A controller release version: `x.y.z`, or `x.y.z-<pre>` with dot-separated+ * alphanumeric identifiers, as a dev build of main publishes+ * (`0.31.1-dev.412.gabc1234`). Build metadata (`+…`) is not used.+ */+export const RELEASE_VERSION = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*))?$/;++type Version = { core: [number, number, number]; pre: string[] };function parseVersion(value: string): Version | null {- const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(value);- return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null;+ const match = RELEASE_VERSION.exec(value);+ if (!match) return null;+ return {+ core: [Number(match[1]), Number(match[2]), Number(match[3])],+ pre: match[4] ? match[4].split('.') : [],+ };+}++/** Whether `value` is a plain `x.y.z` release, not a prerelease. */+export function isPlainVersion(value: string): boolean {+ const parsed = parseVersion(value);+ return parsed !== null && parsed.pre.length === 0;}-/** Whether `candidate` is a later release than `current`; versions not plain x.y.z never are. */+/** Semver precedence of two prerelease identifiers. */+function compareIdentifier(a: string, b: string): number {+ const numericA = /^\d+$/.test(a);+ const numericB = /^\d+$/.test(b);+ if (numericA && numericB) return Number(a) - Number(b);+ if (numericA !== numericB) return numericA ? -1 : 1;+ return a < b ? -1 : a > b ? 1 : 0;+}++/**+ * Whether `candidate` is a later release than `current`, by semver precedence:+ * `0.31.0-dev.3` < `0.31.0-dev.12` < `0.31.0`. A version that is neither form+ * never is.+ */export function isNewer(candidate: string, current: string): boolean {const a = parseVersion(candidate);const b = parseVersion(current);if (!a || !b) return false;- for (let index = 0; index < 3; index++) if (a[index] !== b[index]) return a[index] > b[index];- return false;+ for (let index = 0; index < 3; index++)+ if (a.core[index] !== b.core[index]) return a.core[index]! > b.core[index]!;+ if (a.pre.length === 0 || b.pre.length === 0) return a.pre.length === 0 && b.pre.length > 0;+ for (let index = 0; index < Math.min(a.pre.length, b.pre.length); index++) {+ const order = compareIdentifier(a.pre[index]!, b.pre[index]!);+ if (order !== 0) return order > 0;+ }+ return a.pre.length > b.pre.length;}/**@@ -79,7 +115,7 @@ export async function latestRelease(if (release.draft || release.prerelease || !release.tag_name.startsWith(RELEASE_TAG_PREFIX))continue;const version = release.tag_name.slice(RELEASE_TAG_PREFIX.length);- if (!parseVersion(version)) continue;+ if (!isPlainVersion(version)) continue;const asset = release.assets.find((candidate) => candidate.name === packageAssetName(version));if (!asset) continue;if (!best || isNewer(version, best.version))diff --git a/core/switch_core/config.py b/core/switch_core/config.pyindex af7e3a708..13b8ee85e 100644--- a/core/switch_core/config.py+++ b/core/switch_core/config.py@@ -2,6 +2,7 @@ import osimport reimport sslimport uuid+from dataclasses import dataclassfrom functools import cached_propertyfrom pathlib import Pathfrom typing import Literal@@ -81,6 +82,22 @@ def _require_matching_keypair(# A connection catalog slug (`connections/loader.py`, SLUG_PATTERN)._SERVICE_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")+# The version shape the controller's package script accepts: a release's+# x.y.z, or a dev build's x.y.z-<prerelease> (0.31.1-dev.412.gabc1234).+_CONTROLLER_RELEASE_VERSION_RE = re.compile(+ r"\d+\.\d+\.\d+(?:-[0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*)?"+)+_SHA256_HEX_RE = re.compile(r"[0-9a-f]{64}")+++@dataclass(frozen=True)+class ControllerRelease:+ """The agents controller package every controller is told to run."""++ version: str+ url: str+ sha256: str+class SwitchConfig(BaseSettings):model_config = SettingsConfigDict(env_prefix="", populate_by_name=True)@@ -664,6 +681,15 @@ class SwitchConfig(BaseSettings):# `report_within_s`; a controller that has not reported for three of these# is shown as unknown and refused new placements.controller_status_interval_seconds: int = 60+ # The agents controller release every controller is told to run, in+ # answer to its status report. Set all three or none: with none, no+ # controller is told anything and each keeps the version it has. A+ # controller downloads the package from the URL, refuses it unless its+ # SHA-256 matches, and installs it once no agent session is running.+ # Switch Console's own controller is told nothing: it updates with Console.+ controller_release_version: str | None = None+ controller_release_url: str | None = None+ controller_release_sha256: str | None = None# Share one read of a room's new messages between every client in it# (`transport/room_cache.py`). Without it, a room of N agents reads each# new page N times, which is what exhausted the pool in a restart burst.@@ -781,6 +807,57 @@ class SwitchConfig(BaseSettings):)return self+ @model_validator(mode="after")+ def _validate_controller_release(self) -> "SwitchConfig":+ settings = {+ "CONTROLLER_RELEASE_VERSION": self.controller_release_version,+ "CONTROLLER_RELEASE_URL": self.controller_release_url,+ "CONTROLLER_RELEASE_SHA256": self.controller_release_sha256,+ }+ given = sorted(name for name, value in settings.items() if value)+ if not given:+ return self+ if len(given) != len(settings):+ missing = sorted(set(settings) - set(given))+ raise ValueError(+ "The controller release is set all together or not at all: "+ f"{', '.join(given)} is set but {', '.join(missing)} is not."+ )+ version = self.controller_release_version or ""+ if not _CONTROLLER_RELEASE_VERSION_RE.fullmatch(version):+ raise ValueError(+ "CONTROLLER_RELEASE_VERSION must be x.y.z or x.y.z-<prerelease>, got "+ f"{version!r}."+ )+ url = self.controller_release_url or ""+ parts = urlsplit(url)+ if parts.scheme != "https" or not parts.netloc or url != url.strip():+ raise ValueError(+ f"CONTROLLER_RELEASE_URL must be an https URL with a host, got {url!r}."+ )+ sha256 = self.controller_release_sha256 or ""+ if not _SHA256_HEX_RE.fullmatch(sha256):+ raise ValueError(+ "CONTROLLER_RELEASE_SHA256 must be 64 lowercase hex characters, "+ f"got {sha256!r}."+ )+ return self++ @property+ def controller_release(self) -> ControllerRelease | None:+ """The release controllers are told to run, or None when not set."""+ if not (+ self.controller_release_version+ and self.controller_release_url+ and self.controller_release_sha256+ ):+ return None+ return ControllerRelease(+ version=self.controller_release_version,+ url=self.controller_release_url,+ sha256=self.controller_release_sha256,+ )+@model_validator(mode="after")def _validate_logging(self) -> "SwitchConfig":if self.log_format not in ("text", "json"):diff --git a/core/switch_core/management/controller_routes.py b/core/switch_core/management/controller_routes.pyindex 0bd89f8a6..c8e7987cb 100644--- a/core/switch_core/management/controller_routes.py+++ b/core/switch_core/management/controller_routes.py@@ -270,7 +270,7 @@ async def put_status(principal: PathController,management: Management,session: Session,-) -> dict[str, int]:+) -> dict[str, Any]:if len(await request.body()) > MAX_STATUS_BYTES:raise ManagementError(413,diff --git a/core/switch_core/management/schemas.py b/core/switch_core/management/schemas.pyindex d089a8adb..98d118e49 100644--- a/core/switch_core/management/schemas.py+++ b/core/switch_core/management/schemas.py@@ -19,6 +19,7 @@ from typing import Any, Literal, Selffrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator+from switch_core.config import ControllerReleasefrom switch_core.db.models import (Agent,AgentController,@@ -522,8 +523,46 @@ def connection_view(}+def release_for(+ controller: AgentController, release: ControllerRelease | None+) -> ControllerRelease | None:+ """The release this controller is told to run. None for a Switch Console+ one, whatever is set: the controller Console runs ships with Console and+ updates with it."""+ return None if controller.kind == "console" else release+++def controller_release_view(+ controller: AgentController, release: ControllerRelease | None+) -> dict[str, str] | None:+ """The release the controller is told to run, or null when it is told none."""+ wanted = release_for(controller, release)+ if wanted is None:+ return None+ return {"version": wanted.version, "url": wanted.url, "sha256": wanted.sha256}+++def release_view(+ controller: AgentController, release: ControllerRelease | None+) -> dict[str, Any] | None:+ """Whether the controller runs the release it is told to run: the version+ wanted, the one it last reported, and whether they are the same. Null when+ it is told none."""+ release = release_for(controller, release)+ if release is None:+ return None+ return {+ "wanted": release.version,+ "running": controller.version,+ "current": controller.version == release.version,+ }++def controller_view(- controller: AgentController, state: str, leases: ProcessLeases+ controller: AgentController,+ state: str,+ leases: ProcessLeases,+ release: ControllerRelease | None,) -> dict[str, Any]:return {"id": controller.id,@@ -532,6 +571,7 @@ def controller_view("kind": controller.kind,"platform": controller.platform,"version": controller.version,+ "release": release_view(controller, release),"state": state,"last_seen_at": wire_time_or_none(controller.last_seen_at),"connection": connection_view(controller, leases),diff --git a/core/switch_core/management/service.py b/core/switch_core/management/service.pyindex 36f23e9bb..761d00c72 100644--- a/core/switch_core/management/service.py+++ b/core/switch_core/management/service.py@@ -43,6 +43,7 @@ from switch_core.bridges.agent.protocol.controller_presence import (Binding,ControllerPresence,)+from switch_core.config import ControllerReleasefrom switch_core.db.models import (CONTROLLER_ENROLLMENT_KEY_TYPE,CONTROLLER_KEY_TYPE,@@ -84,6 +85,7 @@ from switch_core.management.schemas import (DefinitionV1,StatusReport,assignment_entry,+ controller_release_view,controller_view,managed_agent_view,operation_view,@@ -119,6 +121,9 @@ class ManagementSettings:# `THIRD_PARTY_AVATARS_ENABLED`. Off, a new agent with no chosen icon# stores none rather than the generated one, which carries its name.third_party_avatars: bool+ # The controller release every controller is told to run; None when the+ # deployment names none, and then controllers are told nothing.+ controller_release: ControllerRelease | None@dataclass(frozen=True)@@ -475,6 +480,9 @@ class ManagementService:return {"assignment_revision": revision,"report_within_s": self.settings.status_interval_seconds,+ "controller_release": controller_release_view(+ controller, self.settings.controller_release+ ),}async def connection_state(@@ -633,7 +641,12 @@ class ManagementService:)leases = await self.leases(session)return [- controller_view(controller, self.state_of(controller, leases), leases)+ controller_view(+ controller,+ self.state_of(controller, leases),+ leases,+ self.settings.controller_release,+ )for controller in controllers]@@ -688,7 +701,10 @@ class ManagementService:if not details:leases = await self.leases(session)view = controller_view(- controller, self.state_of(controller, leases), leases+ controller,+ self.state_of(controller, leases),+ leases,+ self.settings.controller_release,)await session.commit()return view@@ -707,7 +723,12 @@ class ManagementService:session, tenant_id, controller_id, changes)leases = await self.leases(session)- view = controller_view(controller, self.state_of(controller, leases), leases)+ view = controller_view(+ controller,+ self.state_of(controller, leases),+ leases,+ self.settings.controller_release,+ )await session.commit()if "name" in changes:self.presence.rename_controller(controller_id, controller.name)diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.pyindex 6d3f06fbd..c7fb114d1 100644--- a/core/switch_core/management/wiring.py+++ b/core/switch_core/management/wiring.py@@ -33,7 +33,7 @@ from switch_core.bridges.agent.operations.agent_management import ()from switch_core.bridges.agent.protocol.agent_core import AgentCorefrom switch_core.bridges.agent.protocol.controller_presence import ControllerPresence-from switch_core.config import SwitchConfig+from switch_core.config import ControllerRelease, SwitchConfigfrom switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.agent_controller_operation_store import (AgentControllerOperationStore,@@ -129,6 +129,7 @@ def build_management(status_interval_seconds: int,server_url: str | None,third_party_avatars: bool,+ controller_release: ControllerRelease | None,session_factory: async_sessionmaker[AsyncSession],presence: ControllerPresence,auth_cache: ControllerAuthCache,@@ -151,6 +152,7 @@ def build_management(status_interval_seconds=status_interval_seconds,server_url=server_url,third_party_avatars=third_party_avatars,+ controller_release=controller_release,),notifier=ControllerNotifier(),controllers=controllers,@@ -204,6 +206,7 @@ def create_management(status_interval_seconds=config.controller_status_interval_seconds,server_url=config.gateway_public_url,third_party_avatars=config.third_party_avatars_enabled,+ controller_release=config.controller_release,session_factory=session_factory,presence=presence,# The agent API-key cache's bound: the longest a controller revokeddiff --git a/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json b/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.jsonindex 41e63efc5..cf96169a9 100644--- a/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json+++ b/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json@@ -9,6 +9,11 @@"os_version": "6.1.0"},"version": "0.1.0",+ "release": {+ "wanted": "0.2.0",+ "running": "0.1.0",+ "current": false+ },"state": "online","last_seen_at": "2026-10-07T09:00:05Z","connection": {diff --git a/core/tests/switch_core/fixtures/agent_controllers/status_response.json b/core/tests/switch_core/fixtures/agent_controllers/status_response.jsonindex fa66b623e..8229de0d9 100644--- a/core/tests/switch_core/fixtures/agent_controllers/status_response.json+++ b/core/tests/switch_core/fixtures/agent_controllers/status_response.json@@ -1 +1,9 @@-{ "assignment_revision": 3, "report_within_s": 60 }+{+ "assignment_revision": 3,+ "report_within_s": 60,+ "controller_release": {+ "version": "0.2.0",+ "url": "https://github.com/sandbox-quantum/switch/releases/download/switch-agent-controller-v0.2.0/switch-agent-controller-0.2.0.tgz",+ "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"+ }+}diff --git a/core/tests/switch_core/management/harness.py b/core/tests/switch_core/management/harness.pyindex 2ac50fa07..e3a64c8b5 100644--- a/core/tests/switch_core/management/harness.py+++ b/core/tests/switch_core/management/harness.py@@ -45,6 +45,7 @@ from switch_core.bridges.agent.protocol.controller_presence import ()from switch_core.bridges.agent.protocol.controller_stream import IDLEfrom switch_core.bridges.agent.protocol.event_buffer import EventBuffer+from switch_core.config import ControllerReleasefrom switch_core.connections.broker import ServiceBrokerfrom switch_core.db.models import (TENANT_ZERO_ID,@@ -197,6 +198,7 @@ def build_harness(controller_auth_ttl_seconds: float = 5,server_url: str | None = SERVER_URL,third_party_avatars: bool = True,+ controller_release: ControllerRelease | None = None,) -> Harness:"""The controller-token cache is on, as it is by default in production,so every management test runs through it."""@@ -211,6 +213,7 @@ def build_harness(status_interval_seconds=STATUS_INTERVAL,server_url=server_url,third_party_avatars=third_party_avatars,+ controller_release=controller_release,session_factory=session_factory,presence=protocol.connections.controllers,auth_cache=controller_auth_cache,@@ -377,6 +380,36 @@ async def enroll_console()+async def enroll_daemon(+ harness: Harness, client: httpx.AsyncClient, owner: User, name: str = "build-box"+) -> EnrolledController:+ """Enroll a headless controller for `owner` with a one-time code, as+ `switch-agent-controller enroll` does, and exchange its credential."""+ code = await client.post(+ "/gateway/management/enrollment-codes", cookies=cookies_for(owner)+ )+ assert code.status_code == 201, code.text+ body = fixture("enroll_request.json")+ body["proof"]["code"] = code.json()["code"]+ body["controller"]["name"] = name+ enrolled = await client.post("/v1/management/controllers/enroll", json=body)+ assert enrolled.status_code == 201, enrolled.text+ controller_id = enrolled.json()["controller_id"]+ credential = enrolled.json()["credential"]+ token = await client.post(+ f"/v1/management/controllers/{controller_id}/token",+ json={"credential": credential},+ )+ assert token.status_code == 200, token.text+ return EnrolledController(+ controller_id=controller_id,+ credential=credential,+ access_token=token.json()["access_token"],+ owner=owner,+ harness=harness,+ )++async def report_status(client: httpx.AsyncClient,controller: EnrolledController,diff --git a/core/tests/switch_core/management/test_config.py b/core/tests/switch_core/management/test_config.pyindex 919fa00d4..ff0ecb9bc 100644--- a/core/tests/switch_core/management/test_config.py+++ b/core/tests/switch_core/management/test_config.py@@ -104,3 +104,73 @@ def test_capacity_with_agent_management_is_accepted() -> None:hosted_launch_capacity=2,)assert config.hosted_launch_capacity == 2+++_RELEASE = dict(+ controller_release_version="0.31.0",+ controller_release_url="https://example.com/switch-agent-controller-0.31.0.tgz",+ controller_release_sha256="a" * 64,+)+++def test_no_controller_release_by_default() -> None:+ assert _config().controller_release is None+++def test_a_full_controller_release_is_accepted() -> None:+ release = _config(**_RELEASE).controller_release+ assert release is not None+ assert release.version == "0.31.0"+ assert release.url == "https://example.com/switch-agent-controller-0.31.0.tgz"+ assert release.sha256 == "a" * 64+++def test_a_dev_build_controller_release_is_accepted() -> None:+ release = _config(+ **{**_RELEASE, "controller_release_version": "0.31.1-dev.412.gabc1234"}+ ).controller_release+ assert release is not None+ assert release.version == "0.31.1-dev.412.gabc1234"+++def test_empty_controller_release_values_count_as_unset() -> None:+ config = _config(+ controller_release_version="",+ controller_release_url="",+ controller_release_sha256="",+ )+ assert config.controller_release is None++[email protected]("missing", sorted(_RELEASE))+def test_a_partial_controller_release_raises(missing: str) -> None:+ with pytest.raises(ValueError, match="all together or not at all") as caught:+ _config(**{**_RELEASE, missing: None})+ assert missing.upper() in str(caught.value)+++ ("field", "value", "message"),+ [+ ("controller_release_version", "v0.31.0", "CONTROLLER_RELEASE_VERSION"),+ ("controller_release_version", "0.31.0-", "CONTROLLER_RELEASE_VERSION"),+ ("controller_release_version", "0.31.0-dev..1", "CONTROLLER_RELEASE_VERSION"),+ ("controller_release_version", "0.31.0+build", "CONTROLLER_RELEASE_VERSION"),+ ("controller_release_version", "0.31.0\n", "CONTROLLER_RELEASE_VERSION"),+ (+ "controller_release_url",+ "http://example.com/c.tgz",+ "CONTROLLER_RELEASE_URL",+ ),+ ("controller_release_url", "https:///c.tgz", "CONTROLLER_RELEASE_URL"),+ ("controller_release_url", "example.com/c.tgz", "CONTROLLER_RELEASE_URL"),+ ("controller_release_sha256", "A" * 64, "CONTROLLER_RELEASE_SHA256"),+ ("controller_release_sha256", "a" * 63, "CONTROLLER_RELEASE_SHA256"),+ ("controller_release_sha256", "g" * 64, "CONTROLLER_RELEASE_SHA256"),+ ],+)+def test_a_malformed_controller_release_raises(+ field: str, value: str, message: str+) -> None:+ with pytest.raises(ValueError, match=message):+ _config(**{**_RELEASE, field: value})diff --git a/core/tests/switch_core/management/test_contract_fixtures.py b/core/tests/switch_core/management/test_contract_fixtures.pyindex 58ba3e0fa..cbcf5d011 100644--- a/core/tests/switch_core/management/test_contract_fixtures.py+++ b/core/tests/switch_core/management/test_contract_fixtures.py@@ -24,6 +24,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.bridges.agent.commands import room_control_framefrom switch_core.bridges.agent.protocol.event_buffer import Readerfrom switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload+from switch_core.config import ControllerReleasefrom switch_core.db.stores.agent_store import AgentStorefrom switch_core.management import controller_routesfrom switch_core.management.schemas import (@@ -45,6 +46,7 @@ from tests.switch_core.management.harness import (create_managed_agent,definition,enroll_console,+ enroll_daemon,fixture,open_connection,open_stream,@@ -81,9 +83,17 @@ def assert_same_shape(actual: Any, expected: Any, where: str = "$") -> None:)+CONTROLLER_RELEASE = ControllerRelease(+ version="0.2.0",+ url="https://github.com/sandbox-quantum/switch/releases/download/switch-agent-controller-v0.2.0/switch-agent-controller-0.2.0.tgz",+ sha256="9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",+)++@pytest.fixturedef harness(session_factory: async_sessionmaker[AsyncSession]) -> Harness:- return build_harness(session_factory)+ """A release is set, so the fixtures record the fields it adds."""+ return build_harness(session_factory, controller_release=CONTROLLER_RELEASE)class TestRequestFixturesParse:@@ -142,7 +152,7 @@ class TestControllerMessages:async def test_assignment_status_and_operations(self, harness: Harness) -> None:owner = await add_member(harness.session_factory, "ada")async with harness.client() as client:- controller = await enroll_console(harness, client, owner)+ controller = await enroll_daemon(harness, client, owner)await report_status(client, controller, 1, providers=[provider("claude")])created = await create_managed_agent(client,@@ -249,7 +259,7 @@ class TestTheControllersOwnInfo:async def test_set_info(self, harness: Harness) -> None:owner = await add_member(harness.session_factory, "ada")async with harness.client() as client:- controller = await enroll_console(harness, client, owner)+ controller = await enroll_daemon(harness, client, owner)await report_status(client, controller, 1, providers=[provider("claude")], agents=[])diff --git a/core/tests/switch_core/management/test_controller_routes.py b/core/tests/switch_core/management/test_controller_routes.pyindex 65d9a33f8..2f4241c38 100644--- a/core/tests/switch_core/management/test_controller_routes.py+++ b/core/tests/switch_core/management/test_controller_routes.py@@ -14,6 +14,7 @@ from sqlalchemy import selectfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry+from switch_core.config import ControllerReleasefrom switch_core.db.models import AgentControllerEnrollmentCodefrom switch_core.db.stores.api_key_store import ApiKeyStorefrom switch_core.management.bindings import load_bindings@@ -29,9 +30,12 @@ from tests.switch_core.management.harness import (create_managed_agent,definition,enroll_console,+ enroll_daemon,+ ensure_connected,fixture,provider,report_status,+ status_report,)@@ -376,7 +380,12 @@ class TestStatus:"/gateway/management/controllers", cookies=cookies_for(owner))assert response.status_code == 200, response.text- assert response.json() == {"assignment_revision": 0, "report_within_s": 60}+ assert response.json() == {+ "assignment_revision": 0,+ "report_within_s": 60,+ "controller_release": None,+ }+ assert online.json()[0]["release"] is Noneassert online.json()[0]["state"] == "online"assert online.json()[0]["status"]["seq"] == 5assert stale.json()[0]["state"] == "online"@@ -454,6 +463,90 @@ class TestStatus:assert accepted.headers["Switch-Controller-Protocol-Accepts"] == "2-2"+_RELEASE = ControllerRelease(+ version="0.31.0",+ url="https://example.com/switch-agent-controller-0.31.0.tgz",+ sha256="b" * 64,+)+++class TestControllerRelease:+ @pytest.fixture+ def harness(self, session_factory: async_sessionmaker[AsyncSession]) -> Harness:+ return build_harness(session_factory, controller_release=_RELEASE)++ async def test_the_status_answer_names_the_release_and_the_owner_sees_it(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ report = status_report(1)+ report["update"] = {"target": "0.31.0", "state": "waiting_idle"}+ async with harness.client() as client:+ controller = await enroll_daemon(harness, client, owner)+ await ensure_connected(client, controller)+ response = await client.put(+ f"/v1/management/controllers/{controller.controller_id}/status",+ json=report,+ headers=controller.headers,+ )+ listed = await client.get(+ "/gateway/management/controllers", cookies=cookies_for(owner)+ )+ assert response.status_code == 200, response.text+ assert response.json()["controller_release"] == {+ "version": "0.31.0",+ "url": "https://example.com/switch-agent-controller-0.31.0.tgz",+ "sha256": "b" * 64,+ }+ view = listed.json()[0]+ assert view["release"] == {+ "wanted": "0.31.0",+ "running": "0.1.0",+ "current": False,+ }+ assert view["status"]["update"] == {"target": "0.31.0", "state": "waiting_idle"}++ async def test_a_controller_on_the_release_is_current(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ report = status_report(1)+ report["controller"]["version"] = "0.31.0"+ async with harness.client() as client:+ controller = await enroll_daemon(harness, client, owner)+ await ensure_connected(client, controller)+ await client.put(+ f"/v1/management/controllers/{controller.controller_id}/status",+ json=report,+ headers=controller.headers,+ )+ listed = await client.get(+ "/gateway/management/controllers", cookies=cookies_for(owner)+ )+ view = listed.json()[0]+ assert view["version"] == "0.31.0"+ assert view["release"] == {+ "wanted": "0.31.0",+ "running": "0.31.0",+ "current": True,+ }+ assert "update" not in view["status"]++ async def test_a_console_controller_is_told_no_release(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ response = await report_status(client, controller, 1)+ listed = await client.get(+ "/gateway/management/controllers", cookies=cookies_for(owner)+ )+ assert response.status_code == 200, response.text+ assert response.json()["controller_release"] is None+ assert listed.json()[0]["release"] is None++async def _placed_agent(harness: Harness, client, controller: EnrolledController, name: str = "reviewer") -> str:diff --git a/core/tests/test_pin_chart_images.py b/core/tests/test_pin_chart_images.pyindex f74643cd0..2406addb7 100644--- a/core/tests/test_pin_chart_images.py+++ b/core/tests/test_pin_chart_images.py@@ -346,3 +346,45 @@ def test_verify_ignores_a_third_party_image_sharing_a_first_party_name() -> Noneassert pin_chart_images.verify("switch-hosted-controller", rendered, "ghcr.io/acme") == [_pinned("switch-hosted-controller")]+++def _release(created: datetime, tag: str, prerelease: bool = True) -> dict:+ return {+ "id": hash(tag),+ "tag_name": tag,+ "prerelease": prerelease,+ "created_at": created.isoformat().replace("+00:00", "Z"),+ }+++def test_prune_deletes_only_old_dev_controller_prereleases_beyond_the_kept_ones() -> (+ None+):+ dev = [+ _release(+ NOW - timedelta(days=40 + n),+ f"switch-agent-controller-v0.31.1-dev.{9 - n}.g{n:07d}",+ )+ for n in range(4)+ ]+ others = [+ _release(NOW - timedelta(days=90), "switch-agent-controller-v0.31.0", False),+ _release(NOW - timedelta(days=90), "switch-agent-controller-v0.31.0-rc.1"),+ _release(NOW - timedelta(days=90), "switch-v0.31.0-dev.1"),+ _release(+ NOW - timedelta(days=90), "switch-agent-controller-v0.30.0-dev.1", False+ ),+ ]+ young = _release(+ NOW - timedelta(days=1), "switch-agent-controller-v0.31.1-dev.10.gaaaaaaa"+ )++ doomed = prune_dev_packages.doomed_controller_releases(+ [*dev, *others, young], NOW, timedelta(days=30), keep=2+ )++ assert [r["tag_name"] for r in doomed] == [+ "switch-agent-controller-v0.31.1-dev.8.g0000001",+ "switch-agent-controller-v0.31.1-dev.7.g0000002",+ "switch-agent-controller-v0.31.1-dev.6.g0000003",+ ]diff --git a/deploy/hosted/machine/README.md b/deploy/hosted/machine/README.mdindex 80107eeeb..451f182d2 100644--- a/deploy/hosted/machine/README.md+++ b/deploy/hosted/machine/README.md@@ -61,10 +61,10 @@ the workspace gives the machine.linked the controller.- Then, on every boot, it runs `switch-agent-controller install-service--separate-users --no-block` for each seat, with the seat's agents'- directory. That command is idempotent, and the root volume can be new- after an image change. It queues the controller's start: each- controller's unit is ordered after the boot service, so systemd starts it- once the boot completes.+ directory and its releases directory. That command is idempotent, and the+ root volume can be new after an image change. It queues the controller's+ start: each controller's unit is ordered after the boot service, so+ systemd starts it once the boot completes.- A controller that fails to enroll or start does not keep the others fromstarting. The boot tries it again 4 times, 30 s apart, then logs thefailure and completes anyway: failing the boot would stop the other@@ -106,6 +106,7 @@ Index `k`'s ids are fixed, because the data volume outlives the root volume:| data directory | `/data/.switch-controller` | `/data/controllers/<k>/data` || code file | `/data/.switch-controller-code` | `/data/controllers/<k>/code` || agents' directories | `/data/agents` | `/data/controllers/<k>/agents` |+| controller releases | `/data/controller-releases` | `/data/controllers/<k>/releases` |The image bakes index 0's users (`install.sh`). For an index k ≥ 1, the bootcreates the users and groups that are missing with these ids, as `install.sh`@@ -115,6 +116,24 @@ seat's controller unit after the boot service(`/etc/systemd/system/switch-agent-controller-<uid>.service.d/after-boot.conf`)on every boot, since the root volume can be new.+## Controller updates++The image carries one release of the controller, at `/opt/switch/controller`.+Each controller then runs the release Switch names for it, installing it into its+releases directory on the data volume when no session runs and restarting into it+(see "Updates" in the controller's README). An update survives a replaced root+volume, and a controller fix needs no new image.++`install-service --releases-dir` makes the seat's releases directory, owned by+the seat's controller user and readable by everyone, and runs the controller and+its agents through its `current` link. At each boot it points `current` at the+image's release only when `current` is missing, names no readable release, or+names one older than the image's: a boot keeps an update applied since, and an+image with a newer release wins. The boot runs `install-service` (as root) with+the image's CLI only. It runs `status` and `enroll`, as the seat's controller+user, with the release `current` names when there is one, since that release+may have moved the data directory's store on past what the image's reads.+A seat that is in `/data/.switch-controllers.json` but not in the bundle (itsworkspace left the machine) has its controller stopped and disabled, and anyof its agents still running stopped. Its data stays on the volume and itsdiff --git a/deploy/hosted/machine/switch_machine_boot.py b/deploy/hosted/machine/switch_machine_boot.pyindex 914772eab..b09fde051 100644--- a/deploy/hosted/machine/switch_machine_boot.py+++ b/deploy/hosted/machine/switch_machine_boot.py@@ -50,6 +50,9 @@ CONTROLLER_DIR_NAME = ".switch-controller"# The SHA-256 of the code the enrollment in a data directory was made with.CODE_RECORD_NAME = ".switch-controller-code"AGENTS_DIR_NAME = "agents"+RELEASES_DIR_NAME = "controller-releases"+# Where a releases directory's `current` release has the controller's CLI.+CURRENT_CLI = Path("current") / "bin" / "switch-agent-controller"# The other indexes' directories, each in CONTROLLERS_DIR_NAME/<index>.CONTROLLERS_DIR_NAME = "controllers"INDEXES_NAME = ".switch-controllers.json"@@ -129,6 +132,9 @@ class Seat:data_dir: Pathcode_record: Pathagents_dir: Path+ # Where the controller installs the releases Switch names, on the data+ # volume so an update outlives the root volume.+ releases_dir: Path@propertydef gid(self) -> int:@@ -167,6 +173,7 @@ def seat_for(index: int, config: MachineConfig) -> Seat:data_dir=DATA_MOUNT / CONTROLLER_DIR_NAME,code_record=DATA_MOUNT / CODE_RECORD_NAME,agents_dir=DATA_MOUNT / AGENTS_DIR_NAME,+ releases_dir=DATA_MOUNT / RELEASES_DIR_NAME,)home = DATA_MOUNT / CONTROLLERS_DIR_NAME / str(index)return Seat(@@ -176,6 +183,7 @@ def seat_for(index: int, config: MachineConfig) -> Seat:data_dir=home / "data",code_record=home / "code",agents_dir=home / "agents",+ releases_dir=home / "releases",)@@ -669,7 +677,13 @@ def enrolled(commands: Commands, config: MachineConfig, seat: Seat) -> bool:"""Whether the seat's data directory holds an enrollment that was not revoked."""status = commands.result(_as_controller(- seat, [*_cli(config), "status", "--data-dir", str(seat.data_dir)]+ seat,+ [+ *_controller_cli(config, seat),+ "status",+ "--data-dir",+ str(seat.data_dir),+ ],),env=_environment(config),)@@ -684,6 +698,16 @@ def _cli(config: MachineConfig) -> list[str]:return [config.node, config.cli]+def _controller_cli(config: MachineConfig, seat: Seat) -> list[str]:+ """The CLI the seat's controller runs as itself: the release its+ releases directory points at when there is one, which may be newer than+ the image's and have moved its data directory on, else the image's. Only+ ever run as the controller's user: root never runs what that user can+ write."""+ current = seat.releases_dir / CURRENT_CLI+ return [config.node, str(current)] if current.exists() else _cli(config)++def _environment(config: MachineConfig) -> dict[str, str]:return {"PATH": config.path, "LANG": "C.UTF-8"}@@ -738,7 +762,7 @@ def start_controller(_as_controller(seat,[- *_cli(config),+ *_controller_cli(config, seat),"enroll","--server",api_endpoint,@@ -773,6 +797,10 @@ def start_controller(str(seat.agents_dir),"--agent-users",str(config.agent_users),+ # install-service makes it the controller user's, and points its+ # `current` at the image's release unless it names a newer one.+ "--releases-dir",+ str(seat.releases_dir),# The controller's unit is ordered after this service: a restart# waited for here would wait for this very boot to finish."--no-block",diff --git a/deploy/hosted/machine/tests/test_boot.py b/deploy/hosted/machine/tests/test_boot.pyindex 16d322973..26c9bfb66 100644--- a/deploy/hosted/machine/tests/test_boot.py+++ b/deploy/hosted/machine/tests/test_boot.py@@ -636,6 +636,64 @@ def test_a_fresh_machine_enrolls_then_runs_its_agents_as_users_of_their_own(machassert commands.calls.index(enroll) < commands.calls.index(setup)+def test_each_seat_keeps_its_controller_releases_on_the_data_volume(machine):+ commands = FakeCommands({" status ": [(0, "Controller: 9a1c…")]})+ (machine / ".switch-controller").mkdir()+ start(commands, bundle(enrolled_entry(KEY_A)))+ seat = boot.seat_for(2, CONFIG)+ seat.data_dir.mkdir(parents=True)+ start(commands, bundle(enrolled_entry(KEY_A)), index=2)+ first, second = commands.ran("install-service")+ assert first[first.index("--releases-dir") + 1] == str(+ machine / "controller-releases"+ )+ assert second[second.index("--releases-dir") + 1] == str(+ machine / "controllers" / "2" / "releases"+ )+ assert seat.releases_dir == machine / "controllers" / "2" / "releases"+++def test_the_controller_runs_as_itself_from_its_current_release(machine):+ seat = boot.seat_for(0, CONFIG)+ current = seat.releases_dir / "0.31.0" / "bin"+ current.mkdir(parents=True)+ (current / "switch-agent-controller").write_text("")+ (seat.releases_dir / "current").symlink_to(seat.releases_dir / "0.31.0")+ commands = FakeCommands({" status ": [(1, "Not enrolled.")]})+ start(commands)+ released = str(seat.releases_dir / "current" / "bin" / "switch-agent-controller")+ [status] = commands.ran("status")+ [enroll] = commands.ran("enroll")+ assert (+ status[:6]+ == enroll[:6]+ == [+ "/usr/sbin/runuser",+ "-u",+ "switch-controller",+ "--",+ "/opt/switch/node/bin/node",+ released,+ ]+ )+ # Root runs only the image's CLI, never one the controller's user can write.+ [setup] = commands.ran("install-service")+ assert setup[:2] == [+ "/opt/switch/node/bin/node",+ "/opt/switch/controller/bin/switch-agent-controller",+ ]+++def test_a_dangling_current_release_falls_back_to_the_images(machine):+ seat = boot.seat_for(0, CONFIG)+ seat.releases_dir.mkdir()+ (seat.releases_dir / "current").symlink_to(seat.releases_dir / "0.31.0")+ commands = FakeCommands({" status ": [(1, "Not enrolled.")]})+ start(commands)+ [status] = commands.ran("status")+ assert status[5] == "/opt/switch/controller/bin/switch-agent-controller"++def test_an_enrolled_machine_only_starts_its_controller(machine):(machine / ".switch-controller").mkdir()commands = FakeCommands({" status ": [(0, "Controller: 9a1c…")]})diff --git a/deploy/remote/helm/switch/README.md b/deploy/remote/helm/switch/README.mdindex b8288b67d..0a1500f94 100644--- a/deploy/remote/helm/switch/README.md+++ b/deploy/remote/helm/switch/README.md@@ -233,3 +233,26 @@ A deployment that set these variables with `kubectl set env` should remove thembefore the first upgrade that renders them (`kubectl set env deployment/<release>-switch-coreAGENT_MANAGEMENT_ENABLED- CONTROLLER_TOKEN_SECRET-`). Otherwise Helm merges its`valueFrom` into the live `value` and the API server rejects the Deployment.++### Controller updates++`switchCore.controllerRelease` names the agents controller release switch-core+tells every controller to run (cloud machines, and `switch-agent-controller`+on SSH hosts and laptops; Switch Console's own controller updates with Console):++```yaml+switchCore:+ controllerRelease:+ version: "0.31.0"+ url: "https://github.com/sandbox-quantum/switch/releases/download/switch-agent-controller-v0.31.0/switch-agent-controller-0.31.0.tgz"+ sha256: "<sha256 of that file>"+```++A controller running another version downloads the package, refuses it unless+its SHA-256 matches, and installs it and restarts once none of its agents has a+session running. The chart published with a Switch release has these filled in+with that release's controller package, so upgrading Switch upgrades the+controllers too. Set all three or none (the chart refuses to render otherwise);+with none, controllers are told nothing and keep their version. To roll+controllers back, point the three values at the previous release's package and+upgrade.diff --git a/deploy/remote/helm/switch/templates/_helpers.tpl b/deploy/remote/helm/switch/templates/_helpers.tplindex a656f4749..2f80cc2a3 100644--- a/deploy/remote/helm/switch/templates/_helpers.tpl+++ b/deploy/remote/helm/switch/templates/_helpers.tpl@@ -612,6 +612,30 @@ rather than from a CrashLoopBackOff.{{- end -}}{{- end }}+{{/*+Refuse a controller release switch-core would refuse at startup: all three+values or none, an x.y.z or x.y.z-<prerelease> version, an https URL, a lowercase+SHA-256.+*/}}+{{- define "switch.validateControllerRelease" -}}+{{- with .Values.switchCore.controllerRelease }}+{{- if or .version .url .sha256 -}}+{{- if not (and .version .url .sha256) -}}+{{- fail "switchCore.controllerRelease.version, .url and .sha256 are set all together or not at all: switch-core refuses to start with only some of them." -}}+{{- end -}}+{{- if not (regexMatch "^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z]+(\\.[0-9A-Za-z]+)*)?$" .version) -}}+{{- fail (printf "switchCore.controllerRelease.version must be x.y.z or x.y.z-<prerelease>, got %q." .version) -}}+{{- end -}}+{{- if not (regexMatch "^https://[^/?#\\s]+" .url) -}}+{{- fail (printf "switchCore.controllerRelease.url must be an https URL, got %q." .url) -}}+{{- end -}}+{{- if not (regexMatch "^[0-9a-f]{64}$" .sha256) -}}+{{- fail (printf "switchCore.controllerRelease.sha256 must be 64 lowercase hex characters, got %q." .sha256) -}}+{{- end -}}+{{- end -}}+{{- end -}}+{{- end }}+{{/*switch-core container env. Shared by the switch-core Deployment and thepre-upgrade migration Job so they always run against the same configuration@@ -718,6 +742,16 @@ this one. Drop it once the oldest supported image reads ID_SERVER_NAME. */}}- name: CONTROLLER_STATUS_INTERVAL_SECONDSvalue: {{ .Values.switchCore.agentManagement.statusIntervalSeconds | quote }}{{- end }}+{{- with .Values.switchCore.controllerRelease }}+{{- if .version }}+- name: CONTROLLER_RELEASE_VERSION+ value: {{ .version | quote }}+- name: CONTROLLER_RELEASE_URL+ value: {{ .url | quote }}+- name: CONTROLLER_RELEASE_SHA256+ value: {{ .sha256 | quote }}+{{- end }}+{{- end }}- name: GATEWAY_ADMIN_EMAILvalueFrom:secretKeyRef:diff --git a/deploy/remote/helm/switch/templates/switch-core/deployment.yaml b/deploy/remote/helm/switch/templates/switch-core/deployment.yamlindex f47d9bff2..02d4675f6 100644--- a/deploy/remote/helm/switch/templates/switch-core/deployment.yaml+++ b/deploy/remote/helm/switch/templates/switch-core/deployment.yaml@@ -4,6 +4,7 @@{{- include "switch.validateDbTls" . }}{{- include "switch.validateLogging" . }}{{- include "switch.validateObservability" . }}+{{- include "switch.validateControllerRelease" . }}apiVersion: apps/v1kind: Deploymentmetadata:diff --git a/deploy/remote/helm/switch/values.yaml b/deploy/remote/helm/switch/values.yamlindex 22a37ecf3..2d7346f0b 100644--- a/deploy/remote/helm/switch/values.yaml+++ b/deploy/remote/helm/switch/values.yaml@@ -362,6 +362,22 @@ switchCore:# How often a controller must report; one silent for three of these is# shown as unknown and gets no new placements.statusIntervalSeconds: 60+ # The agents controller release switch-core tells every controller to run+ # (CONTROLLER_RELEASE_VERSION / _URL / _SHA256), except Switch Console's own,+ # which updates with Console. A controller that runs+ # another version downloads the package from `url`, refuses it unless its+ # SHA-256 is `sha256`, and installs it and restarts once no agent session+ # is running on it. Set all three or none; with none, controllers are told+ # nothing and keep the version they have. A chart from a Switch release+ # ships with that release's controller package filled in here. To roll+ # controllers back, point these at the previous release's package.+ controllerRelease:+ # x.y.z, or a dev build's x.y.z-<prerelease>.+ version: ""+ # An https URL to the switch-agent-controller-<version>.tgz package.+ url: ""+ # The package's SHA-256, 64 lowercase hex characters.+ sha256: ""# Private hosts switch-core may reach at a URL a tenant or agent supplied —# a Mattermost server, an OpenCode server, an agent icon. Hostnames or CIDRs.# Anything else that is not a public address is refused, and link-local anddiff --git a/docs/design/agent-controllers-v1.md b/docs/design/agent-controllers-v1.mdindex 894496970..9e34f17f4 100644--- a/docs/design/agent-controllers-v1.md+++ b/docs/design/agent-controllers-v1.md@@ -106,7 +106,10 @@ Controller access token (`{id}` must match the token's `cid`, otherwise `403 for- Honours `If-None-Match`; returns `200` with an `ETag` header, or `304`.- `PUT /v1/management/controllers/{id}/status`- Body: a `StatusReport`.- - Returns `{assignment_revision, report_within_s}`.+ - Returns `{assignment_revision, report_within_s, controller_release}`.+ `controller_release` is `{version, url, sha256}`, the controller package the+ deployment names (`CONTROLLER_RELEASE_*`), or null when it names none and for a+ `console` controller, which updates with Switch Console.- An older `seq` is ignored, and the response is still 200.- `GET /v1/management/controllers/{id}/operations?state=pending`- Expired leases are re-offered.@@ -145,7 +148,10 @@ Controller access token (`{id}` must match the token's `cid`, otherwise `403 forshows it open but the process holding it stopped or died), its last `status`, and`workspaces_dir`: the directory the controller makesagents' workspaces in, from that status (`machine.workspaces_dir`), null when it has not- reported one.+ reported one. `release` is `{wanted, running, current}`: the controller version the+ deployment names, the one the controller last reported, and whether they are the+ same; null when the deployment names none, and for a `console` controller. How the controller's move to it is going+ is in its status's `update` (`{target, state, problem}`), when it reports one.- `PATCH /gateway/management/controllers/{id}`- Body: `{name?, description?}`, at least one. Renames the machine and/or changes itsdescription; `description: null` (or blank) clears it. Same limits as at enrollment;diff --git a/docs/design/controller-contract-v1.md b/docs/design/controller-contract-v1.mdindex d0cb141fe..656886037 100644--- a/docs/design/controller-contract-v1.md+++ b/docs/design/controller-contract-v1.md@@ -117,9 +117,32 @@ A full snapshot every time, not a delta. It's sent on every change, and at least```PUT /v1/management/controllers/{id}/statusStatusReport // ≤ 64 KiB- → 200 { assignment_revision: number, report_within_s: number } // tells the controller if it's behind+ → 200 {+ assignment_revision: number, // tells the controller if it's behind+ report_within_s: number,+ controller_release: ControllerRelease | null // the release to run; null = none named+ }```+```ts+type ControllerRelease = {+ version: string // x.y.z, or a dev build's x.y.z-<prerelease>+ url: string // https, the switch-agent-controller-<version>.tgz package+ sha256: string // the package's SHA-256, 64 lowercase hex characters+}+```++`controller_release` is what the deployment sets (`CONTROLLER_RELEASE_VERSION`,+`_URL`, `_SHA256`; Helm `switchCore.controllerRelease`), and the chart from a+Switch release names that release's controller. A controller running another+version downloads the package, refuses it unless its SHA-256 matches, and installs+it and restarts once no agent session is running. It says how that is going in+its report's `update` (`{ target, state, problem? }`), which Core stores with the+rest of the report. Rolling back is pointing the setting at the previous+release's package. A controller older than the field ignores it. A Switch Console+controller (kind `console`) is always told null: it ships with Console and updates+with it.+```tstype StatusReport = {seq: number // monotonic per controller. The server drops anything older@@ -135,6 +158,11 @@ type StatusReport = {providers: ProviderStatus[]tools: ToolStatus[]agents: AgentStatus[]+ update?: { // why it does not run controller_release yet. Absent when it does, and from older controllers+ target: string // the version it is moving to+ state: "waiting_idle" | "failed" | "restart_needed"+ problem?: string // human-readable, when it failed+ }}type ProviderStatus = {diff --git a/docs/old/telemetry-events.md b/docs/old/telemetry-events.mdindex 65dadbdc0..fde594368 100644--- a/docs/old/telemetry-events.md+++ b/docs/old/telemetry-events.md@@ -764,7 +764,7 @@ This is what each carries.| `room_group_deleted` | `room_count` (number); `age_days` (number) || `template_created` | `template_kind` (`room` \| `group` \| `agent` \| `other`) || `template_deleted` | `template_kind` (`room` \| `group` \| `agent` \| `other`); `age_days` (number) |-| `agent_request_refused` | `operation` (`list_templates` \| `get_template` \| `run_template` \| `save_template` \| `update_template` \| `delete_template` \| `create_room` \| `create_room_from_yaml`); `reason` (`not_found` \| `not_yours` \| `name_taken` \| `visibility_not_allowed` \| `invalid` \| `too_large` \| `missing_agents` \| `agent_creation_console_only` \| `busy` \| `run_paused` \| `run_stopped` \| `repeat` \| `kickoff_ignored`) |+| `agent_request_refused` | `operation` (`list_templates` \| `get_template` \| `run_template` \| `save_template` \| `update_template` \| `delete_template` \| `create_room` \| `create_room_from_yaml`); `reason` (`not_found` \| `not_yours` \| `name_taken` \| `visibility_not_allowed` \| `invalid` \| `too_large` \| `has_warnings` \| `missing_agents` \| `agent_creation_console_only` \| `agents_not_created` \| `run_undone` \| `busy` \| `run_paused` \| `run_stopped` \| `repeat` \| `kickoff_ignored`) || `room_link_created` | none || `room_link_removed` | none || `room_role_defined` | `exclusive` (boolean) |diff --git a/gateway/src/data/management.ts b/gateway/src/data/management.tsindex 7d893c5b5..41dba7090 100644--- a/gateway/src/data/management.ts+++ b/gateway/src/data/management.ts@@ -96,6 +96,23 @@ export interface StatusReport {providers: ProviderStatus[];tools: { tool: string; state: string; reason?: string | null }[];agents: AgentStatus[];+ /** Why the controller does not run the release Switch names yet; absent when it does, and from older controllers. */+ update?: ControllerUpdate | null;+}++export interface ControllerUpdate {+ /** The version it is moving to. */+ target: string;+ state: "waiting_idle" | "failed" | "restart_needed";+ problem?: string | null;+}++/** The controller release Switch tells a machine to run, and whether it runs it. */+export interface ControllerReleaseState {+ wanted: string;+ /** The version it last reported; null before it has reported one. */+ running: string | null;+ current: boolean;}export interface Controller {@@ -112,6 +129,8 @@ export interface Controller {/** Null when it has never connected. */connection: ControllerConnection | null;status: StatusReport | null;+ /** Null when Switch names no release for it, as for a Switch Console machine, which updates with Console. */+ release: ControllerReleaseState | null;assignment_revision: number;created_at: string;revoked_at: string | null;diff --git a/gateway/src/pages/machines/MachinesPage.test.tsx b/gateway/src/pages/machines/MachinesPage.test.tsxindex 21302f23f..80b459d9d 100644--- a/gateway/src/pages/machines/MachinesPage.test.tsx+++ b/gateway/src/pages/machines/MachinesPage.test.tsx@@ -20,6 +20,7 @@ const laptop: Controller = {assignment_revision: 3,created_at: "2026-10-01T00:00:00Z",revoked_at: null,+ release: null,status: {seq: 1,observed_at: new Date().toISOString(),@@ -137,6 +138,33 @@ describe("MachinesPage", () => {expect(screen.getByText("failed: invalid_credential")).toBeTruthy();});+ it("shows the controller's version against the release Switch names", async () => {+ const wanted = { wanted: "0.31.0", running: "0.1.0", current: false };+ const machines: Controller[] = [+ { ...laptop, id: "c1", name: "on-it", version: "0.31.0", release: { ...wanted, running: "0.31.0", current: true } },+ { ...laptop, id: "c2", name: "waiting", release: wanted, status: { ...laptop.status!, update: { target: "0.31.0", state: "waiting_idle" } } },+ {+ ...laptop,+ id: "c3",+ name: "stuck",+ release: wanted,+ status: { ...laptop.status!, update: { target: "0.31.0", state: "failed", problem: "SHA-256 mismatch" } },+ },+ { ...laptop, id: "c4", name: "silent", release: wanted },+ ];+ mockManagement({+ "GET /controllers": [200, machines],+ "GET /agents": [200, []],+ "GET /operations": [200, []],+ });+ render(<MachinesPage />);+ expect(await screen.findByText("0.31.0")).toBeTruthy();+ expect(screen.getByText("updating when idle")).toBeTruthy();+ expect(screen.getByText("update failed")).toBeTruthy();+ expect(screen.getByText("behind")).toBeTruthy();+ expect(screen.getAllByText("0.1.0")).toHaveLength(3);+ });+it("shows a revoked machine's agents as not running, whatever it last reported", async () => {const revoked: Controller = {...laptop,diff --git a/gateway/src/pages/machines/MachinesPage.tsx b/gateway/src/pages/machines/MachinesPage.tsxindex b8569d27d..fa06774d1 100644--- a/gateway/src/pages/machines/MachinesPage.tsx+++ b/gateway/src/pages/machines/MachinesPage.tsx@@ -130,6 +130,55 @@ function providerChip(status: ProviderStatus, label: (provider: string) => strin);}+/**+ * The controller's version, and where it stands against the release Switch names: on it,+ * waiting for its sessions to end, or stuck, with why. The versions and the reason are on+ * hover, since a dev build's version is too long to sit in the chip.+ */+function controllerVersion(row: Controller) {+ const running = row.version;+ const release = row.release;+ if (release === null || release.current)+ return (+ <Tooltip+ title={+ release === null+ ? "Switch names no controller release for this machine."+ : `Runs ${release.wanted}, the release Switch names.`+ }+ >+ <span>{running ?? EM_DASH}</span>+ </Tooltip>+ );+ const update = row.status?.update;+ const names = `Runs ${running ?? "an unknown version"}; Switch names ${release.wanted}.`;+ let chip: { label: string; color: "warning" | "error"; why: string };+ if (row.state !== "online" || !update || update.target !== release.wanted)+ chip = {+ label: "behind",+ color: "warning",+ why:+ row.state === "online"+ ? "It has said nothing about moving to it: a controller older than self-update never does, so update it by hand once."+ : "It moves to it once it is connected again.",+ };+ else if (update.state === "waiting_idle")+ chip = { label: "updating when idle", color: "warning", why: "It moves to it once no session is running on it." };+ else if (update.state === "restart_needed")+ chip = { label: "restart needed", color: "warning", why: update.problem ?? "It installed it; restart it to run it." };+ else chip = { label: "update failed", color: "error", why: update.problem ?? "It did not say why." };+ return (+ <Tooltip title={`${names} ${chip.why}`}>+ <Stack direction="row" spacing={1} alignItems="center" sx={{ height: "100%", overflow: "hidden" }}>+ <Typography variant="body2" noWrap>+ {running ?? EM_DASH}+ </Typography>+ <Chip size="small" variant="outlined" color={chip.color} label={chip.label} />+ </Stack>+ </Tooltip>+ );+}+function errorMessage(err: unknown): string {return err instanceof Error ? err.message : String(err);}@@ -261,6 +310,13 @@ export default function MachinesPage() {valueGetter: (_value, row) =>row.platform ? `${row.platform.os} ${row.platform.arch}` : EM_DASH,},+ {+ field: "version",+ headerName: "Controller",+ width: 210,+ valueGetter: (_value, row) => row.version,+ renderCell: ({ row }) => controllerVersion(row),+ },{field: "providers",headerName: "Providers",diff --git a/scripts/prune_dev_packages.py b/scripts/prune_dev_packages.pyindex dfdcd482b..b7391e315 100644--- a/scripts/prune_dev_packages.py+++ b/scripts/prune_dev_packages.py@@ -1,8 +1,9 @@#!/usr/bin/env python3-"""Delete old dev builds from GHCR.+"""Delete old dev builds from GHCR, and the dev controller releases.Every push to main publishes the images and chart under `<owner>/dev/…`-(see .github/workflows/switch-release.yml). Those packages hold nothing but dev+(see .github/workflows/switch-release.yml), and the agents controller as a+GitHub prerelease tagged `switch-agent-controller-v<x.y.z>-dev.<n>.g<sha>`. Those packages hold nothing but devbuilds, so pruning them cannot touch a release. A version is deleted when it isolder than `--max-age-days` AND older than the `--keep` newest tagged builds,so a quiet month does not empty the package and leave the dev environment@@ -14,11 +15,19 @@ moments before it. The cutoff is set an hour before the oldest kept build, so akept index keeps its platform manifests; untagged manifests older than thatbelong to builds being deleted anyway.+The dev controller releases are pruned by the same rule: a prerelease whose+tag names a `-dev.` version is deleted, with its tag, when it is older than+`--max-age-days` and outside the `--keep` newest. Nothing else in the+repository's releases is touched: a release or an RC is never a prerelease+with a dev version.+Needs `gh` authenticated with a token that may delete the packages' versions-(the publishing repository's GITHUB_TOKEN with `packages: write`).+and the repository's releases and tags (the publishing repository's+GITHUB_TOKEN with `packages: write` and `contents: write`).Usage:- python scripts/prune_dev_packages.py --owner <org> --max-age-days 30 --keep 20 [--dry-run]+ python scripts/prune_dev_packages.py --owner <org> --repository <org>/<repo> \\+ --max-age-days 30 --keep 20 [--dry-run]"""from __future__ import annotations@@ -81,11 +90,57 @@ def doomed(]+CONTROLLER_TAG_PREFIX = "switch-agent-controller-v"+++def is_dev_controller_release(release: dict) -> bool:+ tag = release["tag_name"]+ return (+ bool(release.get("prerelease"))+ and tag.startswith(CONTROLLER_TAG_PREFIX)+ and "-dev." in tag[len(CONTROLLER_TAG_PREFIX) :]+ )+++def doomed_controller_releases(+ releases: list[dict], now: datetime, max_age: timedelta, keep: int+) -> list[dict]:+ dev = sorted(+ (r for r in releases if is_dev_controller_release(r)),+ key=_created,+ reverse=True,+ )+ return [r for r in dev[keep:] if _created(r) < now - max_age]+++def _prune_controller_releases(+ repository: str, now: datetime, max_age: timedelta, keep: int, dry_run: bool+) -> None:+ pages = _gh("--paginate", "--slurp", f"/repos/{repository}/releases?per_page=100")+ releases = [release for page in json.loads(pages) for release in page]+ delete = doomed_controller_releases(releases, now, max_age, keep)+ dev = sum(1 for r in releases if is_dev_controller_release(r))+ print(f"{repository} dev controller releases: {dev}, deleting {len(delete)}")+ for release in delete:+ tag = release["tag_name"]+ print(+ f" {'would delete' if dry_run else 'delete'} {tag} {release['created_at']}"+ )+ if not dry_run:+ _gh("--method", "DELETE", f"/repos/{repository}/releases/{release['id']}")+ _gh(+ "--method",+ "DELETE",+ f"/repos/{repository}/git/refs/tags/{quote(tag, safe='')}",+ )++def main(argv: list[str]) -> int:parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)parser.add_argument("--owner", required=True)+ parser.add_argument("--repository", required=True)parser.add_argument("--max-age-days", type=int, required=True)parser.add_argument("--keep", type=int, required=True)parser.add_argument("--dry-run", action="store_true")@@ -119,6 +174,13 @@ def main(argv: list[str]) -> int:"DELETE",f"/orgs/{args.owner}/packages/container/{quote(package, safe='')}/versions/{version['id']}",)+ _prune_controller_releases(+ args.repository,+ now,+ timedelta(days=args.max_age_days),+ args.keep,+ args.dry_run,+ )return 0