Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(devbox): switch instance type on capacity shortage (#124829)

Raúl Negrón-Otero5d55c7e9b80c
diff --git a/.agents/skills/setting-up-devbox/SKILL.md b/.agents/skills/setting-up-devbox/SKILL.md
index dbb2e1b8feda..14234f27e02e 100644
--- a/.agents/skills/setting-up-devbox/SKILL.md
+++ b/.agents/skills/setting-up-devbox/SKILL.md
@@ -73,6 +73,7 @@ This creates the box on first use and resumes it after a stop.
It returns once the box's startup scripts finish, so the box is ready to connect to. `devbox:restart`, `devbox:update` and `devbox:clone` wait the same way. If a script fails or the wait gives up, the box still runs and hogli prints a warning that points to `hogli devbox:logs`.
It brings up the PostHog stack only when the workspace has `--start-app` set, which is covered in [Run the PostHog app](#run-the-posthog-app).
`--disk` (`100` or `200` GiB) applies only when the box is created.
+When AWS has no capacity for the box's instance type in its zone (`InsufficientInstanceCapacity`), hogli starts the box on the template's other instance types by itself and keeps the disk. There is nothing to pass.
`--region` (`us-east-1` or `eu-central-1`) starts that region's default box and creates it if it doesn't exist, so leave it off when resuming an existing box.
A box's region can't change.
diff --git a/docs/internal/coder-workspaces.md b/docs/internal/coder-workspaces.md
index a705a99ea09b..6e06e168541b 100644
--- a/docs/internal/coder-workspaces.md
+++ b/docs/internal/coder-workspaces.md
@@ -120,6 +120,10 @@ Region selection is available for `devbox:start` via `--region` (`us-east-1` or
`devbox:start --start-app` opts a workspace into bringing the PostHog app up (`hogli up`) in the background on every start — useful for scripted, ephemeral devboxes that should be usable without an interactive `hogli up`. The setting is stored as a workspace parameter (`auto_start_app`), so it stays in effect for future starts. The flag applies only when hogli creates the devbox; on an existing devbox hogli refuses it, and the parameter can be changed in the Coder dashboard. Requires a template version that defines the parameter; on older templates hogli drops it with a warning.
+When a build fails with `InsufficientInstanceCapacity`, AWS has no free capacity for the box's instance type in its availability zone.
+`devbox:start`, `devbox:restart` and `devbox:update` then start the box on the template's other instance types in turn, through the Coder build API, because the Coder CLI keeps a parameter's last-build value.
+The template changes the type of the same instance, so the disk stays, and the box keeps the type that started.
+
Runtime commands assume setup is already complete.
If they fail with `Run hogli devbox:setup`, rerun setup on your laptop first.
When in doubt, `hogli devbox:doctor` reports which prerequisite is missing.
diff --git a/tools/hogli-commands/hogli_commands/devbox/cli.py b/tools/hogli-commands/hogli_commands/devbox/cli.py
index 58f92340e942..685cb9d10d19 100644
--- a/tools/hogli-commands/hogli_commands/devbox/cli.py
+++ b/tools/hogli-commands/hogli_commands/devbox/cli.py
@@ -78,6 +78,7 @@ from .coder import (
print_setup_summary,
region_from_workspace_name,
restart_workspace,
+ run_with_capacity_fallback,
server_supports_user_secrets,
share_workspace,
ssh_replace,
@@ -398,10 +399,10 @@ def _start_existing_workspace(
if status == "stopped":
if workspace.get("outdated"):
click.echo(f"Starting devbox '{name}' on the latest template...")
- update_workspace(name, verbose=verbose)
+ run_with_capacity_fallback(name, lambda: update_workspace(name, verbose=verbose))
else:
click.echo(f"Starting devbox '{name}'...")
- start_workspace(name, verbose=verbose)
+ run_with_capacity_fallback(name, lambda: start_workspace(name, verbose=verbose))
_wait_until_ready(name)
click.echo("Started.")
_resume_sync(name)
@@ -410,7 +411,7 @@ def _start_existing_workspace(
click.echo(f"Devbox '{name}' is in state: {status}")
click.echo("Attempting to start...")
- start_workspace(name, verbose=verbose)
+ run_with_capacity_fallback(name, lambda: start_workspace(name, verbose=verbose))
_wait_until_ready(name)
_resume_sync(name)
_print_connection_info(name)
@@ -1518,17 +1519,20 @@ def devbox_start(
config = load_config()
click.echo(f"Creating devbox '{name}' (template={template}, preset={preset}, region={effective_region})...")
- create_workspace(
+ run_with_capacity_fallback(
name,
- disk,
- git_name=config.get("git_name"),
- git_email=config.get("git_email"),
- dotfiles_uri=config.get("dotfiles_uri"),
- region=effective_region,
- template=template,
- preset=preset,
- start_app=start_app,
- verbose=verbose,
+ lambda: create_workspace(
+ name,
+ disk,
+ git_name=config.get("git_name"),
+ git_email=config.get("git_email"),
+ dotfiles_uri=config.get("dotfiles_uri"),
+ region=effective_region,
+ template=template,
+ preset=preset,
+ start_app=start_app,
+ verbose=verbose,
+ ),
)
_wait_until_ready(name)
click.echo("Created.")
@@ -1657,7 +1661,7 @@ def devbox_restart(workspace: str | None, verbose: bool) -> None:
name, workspaces = resolve_workspace_name(workspace)
_get_workspace_or_fail(name, workspaces)
click.echo(f"Restarting '{name}'...")
- restart_workspace(name, verbose=verbose)
+ run_with_capacity_fallback(name, lambda: restart_workspace(name, verbose=verbose))
_wait_until_ready(name)
click.echo("Restarted.")
_print_connection_info(name)
@@ -1675,7 +1679,7 @@ def devbox_update(workspace: str | None, verbose: bool) -> None:
click.echo(f"Devbox '{name}' is already up to date.")
return
click.echo(f"Updating '{name}' to the latest template...")
- update_workspace(name, verbose=verbose)
+ run_with_capacity_fallback(name, lambda: update_workspace(name, verbose=verbose))
_wait_until_ready(name)
click.echo("Updated.")
click.echo(
diff --git a/tools/hogli-commands/hogli_commands/devbox/coder.py b/tools/hogli-commands/hogli_commands/devbox/coder.py
index dc0bb3376949..92383c3393d3 100644
--- a/tools/hogli-commands/hogli_commands/devbox/coder.py
+++ b/tools/hogli-commands/hogli_commands/devbox/coder.py
@@ -22,6 +22,7 @@ import threading
import subprocess
import webbrowser
import urllib.parse
+from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any, NoReturn
@@ -340,6 +341,20 @@ def _run_with_param_retry(
del remaining[dropped]
+# AWS error code when an availability zone has no free capacity for the requested instance type.
+_CAPACITY_ERROR = "InsufficientInstanceCapacity"
+
+
+class CapacityShortage(SystemExit):
+ """A build failed because AWS had no capacity for the box's instance type in its zone."""
+
+
+def _exit_for_failed_build(returncode: int, output: str) -> NoReturn:
+ if _CAPACITY_ERROR in output:
+ raise CapacityShortage(returncode or 1)
+ raise SystemExit(returncode or 1)
+
+
def _resolve_tailscale() -> str | None:
"""Return path to the tailscale CLI, checking PATH then the macOS app bundle."""
if path := shutil.which("tailscale"):
@@ -1375,7 +1390,7 @@ def create_workspace(
]
result = _run_with_param_retry(base_args, parameters, verbose=verbose)
if result.returncode != 0:
- raise SystemExit(result.returncode)
+ _exit_for_failed_build(result.returncode, result.stdout)
# A metadata read over ssh is near-instant; keep the ceiling tight so a wedged
@@ -1494,7 +1509,7 @@ def start_workspace(name: str, *, verbose: bool = False) -> None:
"""Start a stopped workspace."""
result = _run_build(["coder", "start", name, "--yes"], verbose=verbose)
if result.returncode != 0:
- raise SystemExit(result.returncode)
+ _exit_for_failed_build(result.returncode, result.stdout)
# Must exceed the longest blocking startup script timeout in the posthog-linux
@@ -1566,7 +1581,7 @@ def restart_workspace(name: str, *, verbose: bool = False) -> None:
"""
result = _run_build(["coder", "restart", name, "--use-parameter-defaults", "--yes"], verbose=verbose)
if result.returncode != 0:
- raise SystemExit(result.returncode)
+ _exit_for_failed_build(result.returncode, result.stdout)
def update_workspace(name: str, *, verbose: bool = False) -> None:
@@ -1577,15 +1592,155 @@ def update_workspace(name: str, *, verbose: bool = False) -> None:
"""
result = _run_build(["coder", "update", name, "--use-parameter-defaults"], verbose=verbose)
if result.returncode != 0:
- raise SystemExit(result.returncode)
+ _exit_for_failed_build(result.returncode, result.stdout)
# `coder update` only streams the build logs, so it exits 0 when the build fails.
workspace = get_workspace(name)
if workspace is not None and get_workspace_status(workspace) == "failed":
if not verbose:
click.echo(click.style("Build failed. Full output:", fg="red"))
click.echo(result.stdout, nl=False)
+ _exit_for_failed_build(1, result.stdout)
+
+
+# Mutable template parameter. A change to it changes the type of the same EC2
+# instance, so the box keeps its disk.
+INSTANCE_TYPE_PARAMETER = "instance_type"
+BUILD_WAIT_TIMEOUT = 3600
+BUILD_POLL_INTERVAL = 3
+_BUILD_DONE_JOB_STATUSES = frozenset({"succeeded", "failed", "canceled"})
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class _FinishedBuild:
+ job_status: str
+ log_output: str
+
+
+class _CoderApi:
+ """The Coder REST API, called as the user and on the deployment the Coder CLI is logged into.
+
+ The Coder CLI cannot change a mutable parameter on an existing workspace,
+ because it resolves the value to the last build's after any ``--parameter``
+ flag (cli/parameterresolver.go). The build API takes a provided value
+ first, as the Coder dashboard does.
+ """
+
+ def __init__(self) -> None:
+ whoami = _run(["coder", "whoami", "-o", "json"], capture_output=True)
+ if whoami.returncode != 0:
+ _fail(f"Could not read your Coder login. {RUNTIME_SETUP_HINT}", cause="coder_login_unreadable")
+ self._base_url = f"{json.loads(whoami.stdout)[0]['url'].rstrip('/')}/api/v2"
+ self._headers = {"Coder-Session-Token": self._session_token()}
+
+ @staticmethod
+ def _session_token() -> str:
+ # The CLI authenticates with CODER_SESSION_TOKEN when it is set, but
+ # `coder login token` prints only the stored login (cli/login.go).
+ if token := os.environ.get("CODER_SESSION_TOKEN"):
+ return token
+ stored = _run(["coder", "login", "token"], capture_output=True)
+ if stored.returncode != 0:
+ _fail(f"Could not read your Coder login. {RUNTIME_SETUP_HINT}", cause="coder_login_unreadable")
+ return stored.stdout.strip()
+
+ def request(self, method: str, path: str, *, body: dict[str, Any] | None = None) -> Any:
+ try:
+ response = requests.request(method, f"{self._base_url}{path}", headers=self._headers, json=body, timeout=30)
+ except requests.RequestException as error:
+ _fail(f"Could not reach Coder: {error}", cause="coder_api_unreachable")
+ if not response.ok:
+ _fail(f"Coder rejected {method} {path}: {response.text}", cause="coder_api_rejected")
+ return response.json()
+
+ def poll(self, path: str) -> Any | None:
+ """GET ``path``, returning None on a network error or a 5xx so the caller polls again."""
+ try:
+ response = requests.request("GET", f"{self._base_url}{path}", headers=self._headers, json=None, timeout=30)
+ except requests.RequestException:
+ return None
+ if response.status_code >= 500:
+ return None
+ if not response.ok:
+ _fail(f"Coder rejected GET {path}: {response.text}", cause="coder_api_rejected")
+ return response.json()
+
+ def instance_type(self, build_id: str) -> str | None:
+ parameters = self.request("GET", f"/workspacebuilds/{build_id}/parameters")
+ return next((p["value"] for p in parameters if p["name"] == INSTANCE_TYPE_PARAMETER), None)
+
+ def instance_type_options(self, template_version_id: str) -> list[str]:
+ parameters = self.request("GET", f"/templateversions/{template_version_id}/rich-parameters")
+ return next(
+ ([o["value"] for o in p["options"]] for p in parameters if p["name"] == INSTANCE_TYPE_PARAMETER), []
+ )
+
+ def start_on_instance_type(self, workspace_id: str, instance_type: str) -> _FinishedBuild:
+ """Start the workspace on ``instance_type`` and wait for the build to finish."""
+ build = self.request(
+ "POST",
+ f"/workspaces/{workspace_id}/builds",
+ body={
+ "transition": "start",
+ "rich_parameter_values": [{"name": INSTANCE_TYPE_PARAMETER, "value": instance_type}],
+ },
+ )
+ deadline = time.monotonic() + BUILD_WAIT_TIMEOUT
+ while (status := build["job"]["status"]) not in _BUILD_DONE_JOB_STATUSES:
+ if time.monotonic() >= deadline:
+ _fail(f"The devbox build did not finish within {BUILD_WAIT_TIMEOUT}s.", cause="build_wait_timeout")
+ time.sleep(BUILD_POLL_INTERVAL)
+ build = self.poll(f"/workspacebuilds/{build['id']}") or build
+ if status == "succeeded":
+ return _FinishedBuild(job_status=status, log_output="")
+ logs = self.request("GET", f"/workspacebuilds/{build['id']}/logs")
+ # A job that fails outside Terraform, such as a lost provisioner, has its cause only in job.error.
+ output = "".join(f"{log['output']}\n" for log in logs) + (build["job"].get("error") or "")
+ return _FinishedBuild(job_status=status, log_output=output)
+
+
+def run_with_capacity_fallback(name: str, build: Callable[[], None]) -> None:
+ """Run a start build, and on a capacity shortage start the box on the template's other instance types.
+
+ EC2 capacity runs out per availability zone and instance type, so another
+ type in the same zone usually has room. The box stays on the type that
+ started, and a later shortage on it moves the box on again.
+ """
+ try:
+ build()
+ return
+ except CapacityShortage:
+ pass
+
+ workspace = get_workspace(name)
+ if workspace is None:
+ raise SystemExit(1)
+ api = _CoderApi()
+ failed_build = workspace["latest_build"]
+ failed_type = api.instance_type(failed_build["id"])
+ options = api.instance_type_options(failed_build["template_version_id"])
+ if failed_type is None or not options:
raise SystemExit(1)
+ for instance_type in [option for option in options if option != failed_type]:
+ click.echo(
+ click.style(
+ f"AWS has no {failed_type} capacity in the devbox's availability zone. Starting it on {instance_type}...",
+ fg="yellow",
+ )
+ )
+ attempt = api.start_on_instance_type(workspace["id"], instance_type)
+ if attempt.job_status == "succeeded":
+ return
+ if _CAPACITY_ERROR not in attempt.log_output:
+ click.echo(click.style("Build failed. Full output:", fg="red"))
+ click.echo(attempt.log_output, nl=False)
+ raise SystemExit(1)
+ failed_type = instance_type
+ _fail(
+ f"AWS has no capacity in the devbox's availability zone for any of {', '.join(options)}. Try again later.",
+ cause="capacity_exhausted",
+ )
+
def delete_workspace(name: str, *, verbose: bool = False) -> None:
"""Delete a workspace."""
diff --git a/tools/hogli-commands/hogli_commands/tests/test_devbox.py b/tools/hogli-commands/hogli_commands/tests/test_devbox.py
index 3cb044e83271..6e0334af7d2e 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_devbox.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_devbox.py
@@ -10,6 +10,7 @@ import hashlib
import subprocess
from collections.abc import Callable
from pathlib import Path
+from typing import Any
import pytest
from unittest.mock import MagicMock, patch
@@ -1125,6 +1126,31 @@ class TestWorkspaceCreation:
else:
coder.update_workspace("devbox-test-user")
+ @pytest.mark.parametrize(
+ "returncode, build_status, output, shortage",
+ [
+ (1, "failed", "api error InsufficientInstanceCapacity: Insufficient capacity.", True),
+ (1, "failed", "Error: Missing required provider", False),
+ (0, "failed", "api error InsufficientInstanceCapacity: Insufficient capacity.", True),
+ ],
+ ids=["start-capacity", "start-other-error", "update-exits-0-on-capacity"],
+ )
+ def test_failed_build_reports_a_capacity_shortage_only_for_capacity_errors(
+ self, monkeypatch: pytest.MonkeyPatch, returncode: int, build_status: str, output: str, shortage: bool
+ ) -> None:
+ monkeypatch.setattr(
+ coder, "_run_build", lambda args, verbose=False: subprocess.CompletedProcess(args, returncode, output, "")
+ )
+ monkeypatch.setattr(
+ coder, "get_workspace", lambda name: {"name": name, "latest_build": {"status": build_status}}
+ )
+ build = coder.update_workspace if returncode == 0 else coder.start_workspace
+
+ with pytest.raises(SystemExit) as raised:
+ build("devbox-test-user")
+
+ assert isinstance(raised.value, coder.CapacityShortage) is shortage
+
def test_clone_workspace_forwards_source_and_creation_flags(self, monkeypatch: pytest.MonkeyPatch) -> None:
captured: dict[str, object] = {}
monkeypatch.setattr(coder, "_run_build", _fake_run_build_capturing(captured))
@@ -1239,6 +1265,154 @@ class TestWorkspaceCreation:
assert dropped.isdisjoint(_parse_parameter_flags(calls[-1]))
+_CAPACITY_LOG = "api error InsufficientInstanceCapacity: Insufficient capacity."
+
+
+class TestCapacityFallback:
+ @pytest.mark.parametrize(
+ "first_build_shortage, fallback_results, expected_starts, exits",
+ [
+ (False, {}, [], False),
+ (True, {"m7g.4xlarge": ("succeeded", "", "")}, ["m7g.4xlarge"], False),
+ (
+ True,
+ {"m7g.4xlarge": ("failed", _CAPACITY_LOG, ""), "m6g.4xlarge": ("succeeded", "", "")},
+ ["m7g.4xlarge", "m6g.4xlarge"],
+ False,
+ ),
+ (
+ True,
+ {"m7g.4xlarge": ("failed", "", _CAPACITY_LOG), "m6g.4xlarge": ("succeeded", "", "")},
+ ["m7g.4xlarge", "m6g.4xlarge"],
+ False,
+ ),
+ (
+ True,
+ {"m7g.4xlarge": ("failed", _CAPACITY_LOG, ""), "m6g.4xlarge": ("failed", _CAPACITY_LOG, "")},
+ ["m7g.4xlarge", "m6g.4xlarge"],
+ True,
+ ),
+ (True, {"m7g.4xlarge": ("failed", "Error: Missing required provider", "")}, ["m7g.4xlarge"], True),
+ ],
+ ids=[
+ "no-shortage",
+ "next-type-starts",
+ "last-type-starts",
+ "shortage-only-in-job-error",
+ "every-type-short",
+ "other-error-stops",
+ ],
+ )
+ def test_shortage_starts_the_box_on_the_template_s_other_instance_types(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ first_build_shortage: bool,
+ fallback_results: dict[str, tuple[str, str, str]],
+ expected_starts: list[str],
+ exits: bool,
+ ) -> None:
+ starts: list[str] = []
+ build_results: dict[str, tuple[str, str, str]] = {}
+
+ def fake_request(method: str, url: str, **kwargs: Any) -> MagicMock:
+ path = url.removeprefix("https://coder.example.com/api/v2")
+ if path == "/workspacebuilds/b-failed/parameters":
+ body: object = [{"name": "instance_type", "value": "m8g.4xlarge"}, {"name": "repo", "value": _REPO}]
+ elif path == "/templateversions/tv-1/rich-parameters":
+ options = [{"value": t} for t in ("m8g.4xlarge", "m7g.4xlarge", "m6g.4xlarge")]
+ body = [{"name": "instance_type", "options": options}]
+ elif method == "POST" and path == "/workspaces/ws-1/builds":
+ instance_type = kwargs["json"]["rich_parameter_values"][0]["value"]
+ starts.append(instance_type)
+ status, _, job_error = build_results[f"b-{instance_type}"] = fallback_results[instance_type]
+ body = {"id": f"b-{instance_type}", "job": {"status": status, "error": job_error}}
+ elif path.endswith("/logs"):
+ status, log_output, _ = build_results[path.split("/")[2]]
+ assert status != "succeeded", "a successful build must not need its logs"
+ body = [{"output": log_output}]
+ else:
+ raise AssertionError(f"unexpected {method} {path}")
+ return MagicMock(ok=True, json=MagicMock(return_value=body))
+
+ def first_build() -> None:
+ if first_build_shortage:
+ raise coder.CapacityShortage(1)
+
+ monkeypatch.setattr(coder.requests, "request", fake_request)
+ monkeypatch.setattr(
+ coder,
+ "_run",
+ lambda args, **kw: subprocess.CompletedProcess(
+ args, 0, '[{"url": "https://coder.example.com"}]' if "whoami" in args else "token\n", ""
+ ),
+ )
+ monkeypatch.setattr(
+ coder,
+ "get_workspace",
+ lambda name: {"id": "ws-1", "latest_build": {"id": "b-failed", "template_version_id": "tv-1"}},
+ )
+
+ if exits:
+ with pytest.raises(SystemExit):
+ coder.run_with_capacity_fallback("devbox-test-user", first_build)
+ else:
+ coder.run_with_capacity_fallback("devbox-test-user", first_build)
+
+ assert starts == expected_starts
+
+ @pytest.mark.parametrize(
+ "transient",
+ [coder.requests.ConnectionError("reset"), MagicMock(ok=False, status_code=503)],
+ ids=["connection-error", "http-503"],
+ )
+ def test_build_wait_polls_again_after_a_transient_failure(
+ self, monkeypatch: pytest.MonkeyPatch, transient: object
+ ) -> None:
+ polls = iter(
+ [transient, MagicMock(ok=True, status_code=200, json=lambda: {"id": "b-1", "job": {"status": "succeeded"}})]
+ )
+
+ def fake_request(method: str, url: str, **kwargs: Any) -> object:
+ if method == "POST":
+ return MagicMock(ok=True, json=lambda: {"id": "b-1", "job": {"status": "running"}})
+ reply = next(polls)
+ if isinstance(reply, Exception):
+ raise reply
+ return reply
+
+ monkeypatch.setattr(coder.requests, "request", fake_request)
+ monkeypatch.setattr(coder, "BUILD_POLL_INTERVAL", 0)
+ monkeypatch.setattr(coder._CoderApi, "__init__", lambda self: None)
+ api = coder._CoderApi()
+ api._base_url, api._headers = "https://coder.example.com/api/v2", {}
+
+ assert api.start_on_instance_type("ws-1", "m7g.4xlarge").job_status == "succeeded"
+
+ @pytest.mark.parametrize(
+ "env_token, expected", [("env-token", "env-token"), (None, "stored-token")], ids=["env", "stored-login"]
+ )
+ def test_api_authenticates_with_the_token_the_cli_uses(
+ self, monkeypatch: pytest.MonkeyPatch, env_token: str | None, expected: str
+ ) -> None:
+ if env_token is None:
+ monkeypatch.delenv("CODER_SESSION_TOKEN", raising=False)
+ else:
+ monkeypatch.setenv("CODER_SESSION_TOKEN", env_token)
+ monkeypatch.setattr(
+ coder,
+ "_run",
+ lambda args, **kw: subprocess.CompletedProcess(
+ args, 0, '[{"url": "https://coder.example.com"}]' if "whoami" in args else "stored-token\n", ""
+ ),
+ )
+ request = MagicMock(return_value=MagicMock(ok=True, json=MagicMock(return_value=[])))
+ monkeypatch.setattr(coder.requests, "request", request)
+
+ coder._CoderApi().instance_type("b-1")
+
+ assert request.call_args.kwargs["headers"] == {"Coder-Session-Token": expected}
+
+
class TestTemplatePresetResolution:
"""Test the runtime preset resolution that suppresses coder's picker."""