Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(devbox): stop the sync from restarting stopped devboxes (#123432)

Aleksander Błaszkiewiczd03c3daab62b
diff --git a/.agents/skills/setting-up-devbox/SKILL.md b/.agents/skills/setting-up-devbox/SKILL.md
index 54c2a426aa65..dbb2e1b8feda 100644
--- a/.agents/skills/setting-up-devbox/SKILL.md
+++ b/.agents/skills/setting-up-devbox/SKILL.md
@@ -94,6 +94,7 @@ Match what the user asked for:
### 5. Stop when done
`hogli devbox:stop` keeps the disk and stops billing.
+It also pauses the box's `devbox:sync` mirror, and `devbox:start` resumes it.
Stops, starts, and `devbox:update` keep `/home`.
`hogli devbox:destroy` deletes it, so don't keep anything irreplaceable only on a box.
diff --git a/.agents/skills/setting-up-devbox/references/sync.md b/.agents/skills/setting-up-devbox/references/sync.md
index 16ec32de5e6c..c327b0cf7118 100644
--- a/.agents/skills/setting-up-devbox/references/sync.md
+++ b/.agents/skills/setting-up-devbox/references/sync.md
@@ -20,6 +20,8 @@ hogli devbox:sync --terminate # remove the mirror when done
- **Run it locally, from the checkout to mirror.** It walks up from the working directory to find `hogli.yaml` and `.git`, so run it from the repo root being edited, including a worktree. Never run it through `devbox:exec`.
- **Remote-only files stay.** The sync mode is `one-way-safe`, so the box's prewarmed `node_modules`, virtualenv, and `target/` are never deleted. Files that exist on the box but not locally when the mirror is created also stay, such as files the branch deleted, so remove them on the box when a check could pick them up. Lockfiles do sync, so after a lockfile change install dependencies on the box through `devbox:exec` before trusting its checks.
- **A feature branch conflicts on first sync.** A new box is on `master`, so every file the branch changed shows as a conflict in the sync state. Conflicts are per path: other files, including new ones, still sync. Resolve a path, or check out the same branch on the box, only when that file must be mirrored.
+- **A stopped box's mirror waits paused.** `devbox:stop` pauses the mirror, and `devbox:start` resumes it once the box is up. A plain `devbox:sync` leaves a paused mirror paused, so resume one that a box started elsewhere with `--resume`.
+- **The mirror never starts a box.** Its SSH connections run with Coder's autostart off, so a mirror to a box Coder stopped fails until the box runs again.
- **Don't edit mirrored files on the box.** Remote edits fight the local source of truth. `devbox:open --vscode` and `--cursor` warn when a sync is active.
- **Check before recreating.** Compare the box's branch and SHA and the sync state first. If they match and no source file conflicts, keep the existing mirror. Source-file conflicts block reliable QA until resolved. A conflict in box-local config such as `.env` is acceptable when tracked source is clean.
diff --git a/tools/hogli-commands/hogli_commands/devbox/cli.py b/tools/hogli-commands/hogli_commands/devbox/cli.py
index b7d6ec1b9c80..58f92340e942 100644
--- a/tools/hogli-commands/hogli_commands/devbox/cli.py
+++ b/tools/hogli-commands/hogli_commands/devbox/cli.py
@@ -346,6 +346,34 @@ def _render_sync_status(workspace_name: str) -> str:
return click.style(f"● {status}", fg="green")
+def _pause_sync(name: str) -> bool:
+ """Pause the devbox's running sync session while the box is stopped, and return whether it did.
+
+ mutagen redials a lost box forever. Paused, it leaves the box and Coder
+ alone until `devbox:start` resumes it. A failed pause only warns: stopping
+ the box matters more, and the SSH shim already stops the sync from
+ starting it again.
+ """
+ selector = mutagen.workspace_label_selector(name)
+ if all(session.get("paused") for session in mutagen.sync_list(label_selector=selector)):
+ return False
+ try:
+ mutagen.sync_pause(selector)
+ except SystemExit:
+ click.echo(click.style(f"Could not pause the sync to '{name}'. Stopping the box anyway.", fg="yellow"))
+ return False
+ click.echo(f"Paused the sync to '{name}'. `hogli devbox:start` resumes it.")
+ return True
+
+
+def _resume_sync(name: str) -> None:
+ """Resume the devbox's sync session if `devbox:stop` paused it."""
+ selector = mutagen.workspace_label_selector(name)
+ if any(session.get("paused") for session in mutagen.sync_list(label_selector=selector)):
+ mutagen.sync_resume(selector)
+ click.echo(f"Resumed the sync to '{name}'.")
+
+
def _start_existing_workspace(
name: str, workspace: dict[str, Any], *, start_app: bool | None = None, verbose: bool
) -> None:
@@ -376,6 +404,7 @@ def _start_existing_workspace(
start_workspace(name, verbose=verbose)
_wait_until_ready(name)
click.echo("Started.")
+ _resume_sync(name)
_print_connection_info(name)
return
@@ -383,6 +412,7 @@ def _start_existing_workspace(
click.echo("Attempting to start...")
start_workspace(name, verbose=verbose)
_wait_until_ready(name)
+ _resume_sync(name)
_print_connection_info(name)
@@ -1597,13 +1627,22 @@ def devbox_stop(workspace: str | None, verbose: bool) -> None:
name, workspaces = resolve_workspace_name(workspace)
ws = _get_workspace_or_fail(name, workspaces)
+ # Also for a box that Coder stopped on its schedule, whose sync still redials it.
+ paused = _pause_sync(name)
+
status = get_workspace_status(ws)
if status == "stopped":
click.echo(f"Devbox '{name}' is already stopped.")
return
click.echo(f"Stopping '{name}'...")
- stop_workspace(name, verbose=verbose)
+ try:
+ stop_workspace(name, verbose=verbose)
+ except (Exception, SystemExit):
+ if paused:
+ mutagen.sync_resume(mutagen.workspace_label_selector(name))
+ click.echo(f"The box still runs, so the sync to '{name}' is resumed.")
+ raise
start_command = f"hogli devbox:start {workspace}" if workspace else "hogli devbox:start"
click.echo("Stopped. Billing for the box stops now.")
click.echo(f"The disk and your files stay until you destroy the box. Run '{start_command}' to resume it.")
diff --git a/tools/hogli-commands/hogli_commands/devbox/mutagen.py b/tools/hogli-commands/hogli_commands/devbox/mutagen.py
index 38f81bdb3ab2..05e7acf20e86 100644
--- a/tools/hogli-commands/hogli_commands/devbox/mutagen.py
+++ b/tools/hogli-commands/hogli_commands/devbox/mutagen.py
@@ -35,24 +35,30 @@ _MUTAGEN_VERSION = "0.18.1"
# devbox connections: a shim on MUTAGEN_SSH_PATH bumps the count so the
# connection rides out a reset window. _KEEPALIVE_COUNT x interval(10s) is the
# resulting silence tolerance (~30s, comfortably past the ~26s reset cadence).
+# The shim also turns off Coder's SSH autostart. mutagen redials a lost box forever,
+# and each redial to a stopped box would otherwise queue a template dry run and a start build.
_SSH_SHIM_DIR = Path.home() / ".hogli" / "mutagen-ssh-shim"
_KEEPALIVE_COUNT = 3
_CONNECT_TIMEOUT_SECONDS = 60
# Baked at generation time (see ensure_ssh_shim): only when the ssh/scp target is
# a devbox (a `coder.*` host -- the sole thing hogli syncs to) do we bump
-# mutagen's count; every other invocation passes through untouched, so a shared
+# mutagen's count and turn off autostart; every other invocation passes through untouched, so a shared
# daemon's non-devbox syncs are unaffected. Execs the real ssh/scp by absolute
# path so a shim-dir entry on PATH can never make the shim re-invoke itself.
_SHIM_TEMPLATE = """#!/usr/bin/env bash
# Managed by hogli (`hogli devbox:sync`) -- do not edit; regenerated on sync.
# Raises mutagen's intolerant `-oServerAliveCountMax=1` for devbox (coder.*)
# connections so the sync survives a Tailscale/DERP path reset instead of dying
-# with a broken pipe. Any non-devbox ssh is passed through unchanged.
+# with a broken pipe, and stops those connections from starting a stopped box.
+# Any non-devbox ssh is passed through unchanged.
devbox=
for a in "$@"; do
case "$a" in coder.*|*@coder.*) devbox=1 ;; esac
done
+if [ -n "$devbox" ]; then
+ export CODER_SSH_DISABLE_AUTOSTART=true
+fi
args=()
for a in "$@"; do
case "$a" in
@@ -259,12 +265,13 @@ def _write_owner_only(target: Path, content: str) -> None:
def ensure_ssh_shim() -> Path:
- """Write the ssh/scp keepalive-rewriting shims, returning their directory.
+ """Write the ssh/scp shims (keepalive rewrite, no Coder autostart), returning their directory.
Idempotent: each shim is rewritten only when its content (which embeds the
resolved real-binary path and keepalive count) changes. mutagen invokes both
- ssh and scp, so both are shimmed; the rewrite is a no-op for any arg that
- isn't ``-oServerAliveCountMax=``.
+ ssh and scp, so both are shimmed; for a devbox host they rewrite the
+ keepalive and connect-timeout args and set ``CODER_SSH_DISABLE_AUTOSTART``,
+ and any other invocation passes through unchanged.
"""
_SSH_SHIM_DIR.mkdir(parents=True, exist_ok=True)
# The daemon execs these scripts, so keep the dir and scripts owner-only: no
diff --git a/tools/hogli-commands/hogli_commands/tests/test_devbox.py b/tools/hogli-commands/hogli_commands/tests/test_devbox.py
index d3f32b0777ea..3cb044e83271 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_devbox.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_devbox.py
@@ -2421,6 +2421,10 @@ class TestDevboxSetupGate:
class TestStartExistingWorkspace:
+ @pytest.fixture(autouse=True)
+ def _no_sync(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(devbox_mutagen, "sync_list", lambda **kw: [])
+
@pytest.mark.parametrize(
"outdated, expected", [(False, ["start"]), (True, ["update"])], ids=["current", "outdated"]
)
@@ -2447,6 +2451,91 @@ class TestStartExistingWorkspace:
"devbox-test-user", {"latest_build": {"status": "stopped"}}, start_app=True, verbose=False
)
+ @pytest.mark.parametrize(
+ "status, paused, expected",
+ [
+ ("stopped", True, ["start", "resume"]),
+ ("stopped", False, ["start"]),
+ ("running", True, []),
+ ],
+ ids=["stopped-paused", "stopped-unpaused", "running-paused"],
+ )
+ def test_resumes_the_sync_devbox_stop_paused_once_the_box_started(
+ self, monkeypatch: pytest.MonkeyPatch, status: str, paused: bool, expected: list[str]
+ ) -> None:
+ calls: list[str] = []
+ monkeypatch.setattr(devbox_cli, "start_workspace", lambda name, verbose=False: calls.append("start"))
+ monkeypatch.setattr(devbox_cli, "extract_workspace_label", lambda name: None)
+ monkeypatch.setattr(devbox_mutagen, "sync_list", lambda **kw: [{"name": "ph-x", "paused": paused}])
+ monkeypatch.setattr(devbox_mutagen, "sync_resume", lambda selector: calls.append("resume"))
+
+ devbox_cli._start_existing_workspace("devbox-test-user", {"latest_build": {"status": status}}, verbose=False)
+
+ assert calls == expected
+
+
+class TestDevboxStop:
+ @pytest.fixture(autouse=True)
+ def _stubs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(devbox_cli, "ensure_runtime_ready", lambda: None)
+ monkeypatch.setattr(devbox_cli, "resolve_workspace_name", lambda ws, **kw: ("devbox-test-user", []))
+
+ @pytest.mark.parametrize(
+ "status, sessions, pause_fails, stop_fails, expected, exit_code",
+ [
+ ("running", [{"paused": False}], False, False, ["pause", "stop"], 0),
+ ("stopped", [{"paused": False}], False, False, ["pause"], 0),
+ ("running", [], False, False, ["stop"], 0),
+ ("running", [{"paused": True}], False, False, ["stop"], 0),
+ ("running", [{"paused": False}], True, False, ["pause", "stop"], 0),
+ ("running", [{"paused": False}], False, True, ["pause", "stop", "resume"], 1),
+ ("running", [{"paused": True}], False, True, ["stop"], 1),
+ ],
+ ids=[
+ "running",
+ "already-stopped",
+ "no-sync",
+ "sync-already-paused",
+ "pause-fails-box-still-stops",
+ "stop-fails-sync-resumes",
+ "stop-fails-user-pause-kept",
+ ],
+ )
+ def test_pauses_the_sync_around_the_stop(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ status: str,
+ sessions: list[dict[str, bool]],
+ pause_fails: bool,
+ stop_fails: bool,
+ expected: list[str],
+ exit_code: int,
+ ) -> None:
+ calls: list[str] = []
+
+ def record(call: str, fails: bool = False) -> Callable[..., None]:
+ def run(*args: object, **kwargs: object) -> None:
+ calls.append(call)
+ if fails:
+ raise SystemExit(1)
+
+ return run
+
+ monkeypatch.setattr(
+ devbox_cli,
+ "get_workspace",
+ lambda name, workspaces=None: {"name": name, "latest_build": {"status": status}},
+ )
+ monkeypatch.setattr(devbox_mutagen, "sync_list", lambda **kw: sessions)
+ monkeypatch.setattr(devbox_mutagen, "sync_pause", record("pause", pause_fails))
+ monkeypatch.setattr(devbox_mutagen, "sync_resume", record("resume"))
+ monkeypatch.setattr(devbox_cli, "stop_workspace", record("stop", stop_fails))
+
+ result = runner.invoke(cli, ["devbox:stop"])
+
+ assert result.exit_code == exit_code, result.output
+ assert calls == expected
+
class TestDevboxList:
"""Test the devbox:list command."""
@@ -3688,6 +3777,34 @@ class TestKeepaliveShim:
# Non-devbox ssh must pass through byte-for-byte, keepalive included.
assert forwarded == incoming
+ @pytest.mark.parametrize(
+ "target, disabled",
+ [("coder.devbox-test-user", True), ("[email protected]", False)],
+ ids=["devbox-host-disabled", "non-devbox-host-untouched"],
+ )
+ def test_shim_disables_coder_autostart_only_for_devbox_hosts(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, target: str, disabled: bool
+ ) -> None:
+ shim_dir = tmp_path / "shim"
+ log = tmp_path / "env.log"
+ fake = tmp_path / "fakessh"
+ fake.write_text(
+ "#!/usr/bin/env bash\nprintf '%s' \"${CODER_SSH_DISABLE_AUTOSTART-unset}\" > '" + str(log) + "'\n"
+ )
+ fake.chmod(0o755)
+
+ monkeypatch.setattr(devbox_mutagen, "_SSH_SHIM_DIR", shim_dir)
+ monkeypatch.setattr(devbox_mutagen, "_resolve_real_ssh", lambda name: str(fake))
+ devbox_mutagen.ensure_ssh_shim()
+
+ env = {k: v for k, v in os.environ.items() if k != "CODER_SSH_DISABLE_AUTOSTART"}
+ result = subprocess.run(
+ [str(shim_dir / "ssh"), "-oServerAliveCountMax=1", target, "true"], capture_output=True, text=True, env=env
+ )
+
+ assert result.returncode == 0
+ assert log.read_text() == ("true" if disabled else "unset")
+
def test_ensure_ssh_shim_writes_both_executables_and_is_idempotent(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None: