Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(regression): hold environments during long-running work (#2886)

Call White3dbda5077781
diff --git a/AGENTS.md b/AGENTS.md
index 2a4affb19..aeceb81f2 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -126,6 +126,10 @@ Hard rules:
explicitly with
`python3 scripts/incus_regression.py delete --target worktree --slug <slug> --yes`
when merged, abandoned, or stale
+- before proposing or performing deletion of any regression environment, check
+ its hold with `status` or `reconcile`; if held, tell the user who holds it and
+ why, and do not offer a one-click delete. Long-running work must hold its
+ environment for as long as it runs.
- `reconcile` reports every worktree environment held by Incus, including
environments created outside the runner, and only forgets metadata for
environments that are no longer present. It never deletes an environment.
diff --git a/docs/regression/README.md b/docs/regression/README.md
index b11558100..d9aa272f1 100644
--- a/docs/regression/README.md
+++ b/docs/regression/README.md
@@ -194,6 +194,26 @@ python3 scripts/incus_regression.py reconcile
python3 scripts/incus_regression.py reconcile --yes
```
+Protect a worktree environment while long-running work uses it, then release
+it when that work finishes:
+
+```bash
+python3 scripts/incus_regression.py hold --target worktree --slug <slug> --owner <session-id-or-name> --reason "Long-running evaluation"
+python3 scripts/incus_regression.py status --target worktree --slug <slug>
+python3 scripts/incus_regression.py release --target worktree --slug <slug>
+```
+
+The instance itself stores `user.avibe-regression.hold.owner`, `.hold.reason`,
+and `.hold.since` (UTC), so another checkout sees the same hold. `hold` sets or
+replaces it; `release` clears only these fields. `status` and `reconcile` show
+the owner, reason, and time, including for instances without runner metadata.
+`delete` refuses a held environment before removing anything, even with
+`--yes`; only an explicit `--ignore-hold` overrides it. Deletion previews
+(`--dry-run`) also check the live hold and require a reachable Incus daemon
+unless explicitly overridden. Reconcile never includes that flag in its
+printed commands. Holds do not expire automatically or stop
+`up`/`down`, and old runners or raw Incus commands do not enforce them.
+
Fresh-install worktree environment, for testing the first-run experience:
```bash
@@ -221,8 +241,12 @@ config and state to rerun the wizard while keeping CLI logins made in it. The
master environment is always seeded.
Delete worktree environments promptly after the worktree is merged, abandoned,
-or removed. The persistent `master` environment should stay running and preserve
-its product state across normal source updates.
+or removed. Before proposing or performing deletion of any regression
+environment, check its hold with `status` or `reconcile`; if held, tell the user
+who holds it and why, and do not offer a one-click delete. Long-running work
+must hold its environment for as long as it runs. The persistent `master`
+environment should stay running and preserve its product state across normal
+source updates.
`reconcile` answers "what is actually still here?". It enumerates worktree
environments from Incus rather than from the runner's metadata, so an
diff --git a/scripts/incus_regression.py b/scripts/incus_regression.py
index 9867ba2b4..768299f60 100644
--- a/scripts/incus_regression.py
+++ b/scripts/incus_regression.py
@@ -101,6 +101,7 @@ SEED_MODES = (SEED_REGRESSION, SEED_NONE)
# has rebuilt the home, it matches neither mode.
SEED_SWITCHING = "switching"
SEED_CONFIG_KEY = "user.avibe-regression.seed"
+HOLD_CONFIG_KEYS = tuple(f"user.avibe-regression.hold.{field}" for field in ("owner", "reason", "since"))
SHOW_RUNTIME_BUILD_TIMEOUT_SECONDS = 300
SLUG_RE = re.compile(r"^[a-z][a-z0-9-]{1,38}[a-z0-9]$")
DAEMON_UNREACHABLE_HINT = (
@@ -280,6 +281,19 @@ def instance_record(runner: Runner, remote: str | None, project: str, instance:
return None
+def describe_instance_hold(record: dict | None) -> str | None:
+ """Read the hold from the instance, never from this checkout's metadata."""
+ config = (record or {}).get("config") or {}
+ if not isinstance(config, dict):
+ raise RegressionError("Could not read the instance config to check its regression hold.")
+ values = [str(config.get(key) or "").strip() for key in HOLD_CONFIG_KEYS]
+ if not any(values):
+ return None
+ # A partial marker still protects the environment until explicitly released.
+ owner, reason, since = (value or "unknown" for value in values)
+ return f"HELD: owner={owner}; reason={reason}; since={since}"
+
+
def instance_exists(runner: Runner, remote: str | None, project: str, instance: str) -> bool:
return instance_record(runner, remote, project, instance) is not None
@@ -422,6 +436,9 @@ def target_update_lock(repo_root: Path, remote: str | None, project: str, *, dry
of them creates, destroys or forgets either half; a `down` that interrupts a
build makes that `up` fail, and its reservation gives the row back.
+ `hold` and `release` also take this lock so their marker changes cannot race
+ a `delete` between its hold check and removal of the instance.
+
`blocking=False` is for a caller whose answer to "somebody else holds this"
is to stop rather than to wait. The acquire is then both the proof that
nobody holds it and the protection, which is the only order with no window
@@ -1045,6 +1062,7 @@ class ObservedInstance:
project: str
state: str
+ hold: str | None = None
@dataclass(frozen=True)
@@ -1169,6 +1187,7 @@ def worktree_instances(runner: Runner, *, remote: str | None) -> dict[str, tuple
ObservedInstance(
project=str(item.get("project") or "default"),
state=str(item.get("status") or "Unknown"),
+ hold=describe_instance_hold(item),
)
)
return {name: tuple(observed) for name, observed in instances.items()}
@@ -2970,6 +2989,9 @@ def cmd_status(args: argparse.Namespace) -> int:
if not args.dry_run:
require_incus()
runner = Runner(dry_run=args.dry_run)
+ hold = describe_instance_hold(instance_record(runner, args.remote, target.project, target.instance))
+ if hold:
+ print(f"{target.project}/{target.instance}: {hold}")
failed = 0
for command in (
incus("list", *optional_remote_ref(args.remote), project=target.project),
@@ -3014,12 +3036,39 @@ def cmd_down(args: argparse.Namespace) -> int:
return 0
+def cmd_hold(args: argparse.Namespace) -> int:
+ """Set or release an instance-local hold without changing its running work."""
+ releasing = args.command == "release"
+ values = ("", "", "") if releasing else (args.owner.strip(), args.reason.strip(), datetime.now(timezone.utc).isoformat())
+ if not releasing and not all(values):
+ raise RegressionError("A hold requires a non-empty --owner and --reason.")
+ repo_root = current_repo_root()
+ load_env_file(repo_root, args.env_file)
+ target = resolve_target(args, repo_root, dry_run=args.dry_run, allocate_port=False)
+ if not args.dry_run:
+ require_incus()
+ with target_update_lock(repo_root, args.remote, target.project, dry_run=args.dry_run, blocking=False):
+ runner = Runner(dry_run=args.dry_run)
+ if not args.dry_run and instance_record(runner, args.remote, target.project, target.instance) is None:
+ raise RegressionError(f"Regression instance {target.project}/{target.instance} does not exist.")
+ # One config update for all three fields; empty values clear the marker.
+ runner.run(incus(
+ "config", "set", remote_ref(args.remote, target.instance),
+ *(f"{key}={value}" for key, value in zip(HOLD_CONFIG_KEYS, values)),
+ project=target.project,
+ ))
+ action = "Release hold" if releasing else f"Hold for {values[0]}: {values[1]} (since {values[2]})"
+ print(f"{'Would apply' if args.dry_run else 'Applied'} to {target.project}/{target.instance}: {action}")
+ return 0
+
+
def cmd_delete(args: argparse.Namespace) -> int:
repo_root = current_repo_root()
load_env_file(repo_root, args.env_file)
if args.target == MASTER_TARGET and not args.yes:
raise RegressionError("Deleting the master regression environment requires --yes.")
- if not args.dry_run:
+ needs_incus = not args.dry_run or not getattr(args, "ignore_hold", False)
+ if needs_incus:
require_incus()
# Removing the objects and forgetting the row are one change to what this
# slug names, so both happen under the environment's update lock -- the same
@@ -3035,6 +3084,17 @@ def cmd_delete(args: argparse.Namespace) -> int:
with target_update_lock(repo_root, args.remote, lock_project, dry_run=args.dry_run, blocking=False):
target = resolve_target(args, repo_root, dry_run=args.dry_run, allocate_port=False, slug=slug)
runner = Runner(dry_run=args.dry_run)
+ if not getattr(args, "ignore_hold", False):
+ # A dry run still needs a real, read-only daemon listing to prove
+ # that a hold is absent; the mutation runner intentionally reports
+ # no records in dry-run mode.
+ hold_runner = Runner(dry_run=False) if args.dry_run else runner
+ hold = describe_instance_hold(instance_record(hold_runner, args.remote, target.project, target.instance))
+ if hold:
+ raise RegressionError(
+ f"Refusing to delete {target.slug}: {hold}. "
+ "Use --ignore-hold to explicitly override this hold."
+ )
results = (
runner.run(incus("delete", remote_ref(args.remote, target.instance), "--force", project=target.project), check=False),
runner.run(incus("project", "delete", remote_ref(args.remote, target.project)), check=False),
@@ -3122,6 +3182,9 @@ def cmd_reconcile(args: argparse.Namespace) -> int:
print(f"{len(live)} worktree regression environment(s) exist in {authority}:")
for env in live:
print(f" {env.slug} [{env.footprint}] {describe_worktree_entry(env.entry)}")
+ for item in env.instances:
+ if item.hold:
+ print(f" {item.project}/{env.instance}: {item.hold}")
print()
# Deletion stays explicit and per-environment. `delete` derives the
# project and instance from the slug by naming convention, so it
@@ -3136,7 +3199,7 @@ def cmd_reconcile(args: argparse.Namespace) -> int:
# Saying both is the only honest report.
deletable = [env for env in live if env.deletable_by_slug]
if deletable:
- print("Delete any of them with:")
+ print("Explicit deletion commands (held environments are refused):")
for env in deletable:
print(
" python3 scripts/incus_regression.py delete --target worktree "
@@ -3299,12 +3362,18 @@ def build_parser() -> argparse.ArgumentParser:
("shell", cmd_shell),
("down", cmd_down),
("delete", cmd_delete),
+ ("hold", cmd_hold),
+ ("release", cmd_hold),
):
sub = subparsers.add_parser(name)
add_common(sub)
add_target_args(sub)
if name == "delete":
sub.add_argument("--yes", action="store_true")
+ sub.add_argument("--ignore-hold", action="store_true", help="Explicitly override an environment's hold.")
+ if name == "hold":
+ sub.add_argument("--owner", required=True, help="Session id or name holding this environment.")
+ sub.add_argument("--reason", required=True, help="Why this environment must be kept.")
sub.set_defaults(func=func)
reconcile = subparsers.add_parser(
diff --git a/tests/test_incus_regression.py b/tests/test_incus_regression.py
index 9a8b52850..c6e1244bd 100644
--- a/tests/test_incus_regression.py
+++ b/tests/test_incus_regression.py
@@ -11,6 +11,7 @@ import stat
import subprocess
import sys
from collections.abc import Sequence
+from datetime import datetime, timezone
from pathlib import Path
import pytest
@@ -2563,6 +2564,173 @@ def reconcile_fixture(
return runtime / "worktrees.json", commands
+def hold_fixture(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, target: str = "worktree"):
+ """Fake Incus config storage; run the real CLI without any host environment."""
+ project, instance = ("avr-master", "avibe-master") if target == "master" else ("avr-wt-eval", "avibe-wt-eval")
+ config = {"user.avibe-regression.seed": "regression"}
+ commands = []
+ _, listing = daemon_listing(project, instance, config=config)
+
+ class RecordingRunner(incus_regression.Runner):
+ def records(self, command, *, what):
+ return [] if self.dry_run else listing(self, command, what=what)
+
+ def run(self, command, **kwargs):
+ if self.dry_run:
+ return super().run(command, **kwargs)
+ commands.append(command)
+ if command[3:5] == ["config", "set"]:
+ for setting in command[6:]:
+ key, value = setting.split("=", 1)
+ config[key] = value
+ return subprocess.CompletedProcess(command, 0, "", "")
+
+ monkeypatch.setenv("INCUS_CMD", "incus")
+ monkeypatch.setattr(incus_regression, "Runner", RecordingRunner)
+ monkeypatch.setattr(incus_regression, "current_repo_root", lambda: tmp_path)
+ monkeypatch.setattr(incus_regression, "git_common_root", lambda root: root)
+ monkeypatch.setattr(incus_regression, "load_env_file", lambda *args: None)
+ monkeypatch.setattr(incus_regression, "require_incus", lambda: None)
+ return config, commands
+
+
[email protected]("target", ["worktree", "master"])
[email protected]("dry_run", [False, True], ids=["apply", "dry-run"])
+def test_hold_then_status_and_release_preserve_unrelated_config(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], target: str, dry_run: bool
+) -> None:
+ """The CLI persists/shows/releases one instance hold; existing seed tests never cover this marker."""
+ config, commands = hold_fixture(tmp_path, monkeypatch, target)
+ original = dict(config)
+ flags = ["--target", target, "--slug", "eval"] + (["--dry-run"] if dry_run else [])
+ before = datetime.now(timezone.utc)
+ assert incus_regression.main(["hold", *flags, "--owner", "session-one", "--reason", "Evaluation in progress"]) == 0
+ after = datetime.now(timezone.utc)
+ capsys.readouterr()
+ assert incus_regression.main(["status", *flags]) == 0
+ out = capsys.readouterr().out
+ if dry_run:
+ assert config == original
+ assert commands == []
+ else:
+ since = config["user.avibe-regression.hold.since"]
+ assert before <= datetime.fromisoformat(since) <= after
+ assert since.endswith("+00:00")
+ assert config == {
+ **original,
+ "user.avibe-regression.hold.owner": "session-one",
+ "user.avibe-regression.hold.reason": "Evaluation in progress",
+ "user.avibe-regression.hold.since": since,
+ }
+ assert f"HELD: owner=session-one; reason=Evaluation in progress; since={since}" in out
+ assert len([command for command in commands if "set" in command]) == 1
+ assert incus_regression.main(["release", *flags]) == 0
+ assert {key: value for key, value in config.items() if value} == original
+ capsys.readouterr()
+ assert incus_regression.main(["status", *flags]) == 0
+ assert "HELD:" not in capsys.readouterr().out
+
+
[email protected]("target", ["worktree", "master"])
[email protected]("dry_run", [False, True], ids=["apply", "dry-run"])
+def test_delete_held_environment_refuses_before_mutation_unless_explicitly_overridden(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], target: str, dry_run: bool
+) -> None:
+ """A persistent hold protects running work, including previews that must not offer its deletion."""
+ config, commands = hold_fixture(tmp_path, monkeypatch, target)
+ config.update({
+ "user.avibe-regression.hold.owner": "session-one",
+ "user.avibe-regression.hold.reason": "Evaluation in progress",
+ "user.avibe-regression.hold.since": "2026-10-20T12:00:00+00:00",
+ })
+ original = dict(config)
+ flags = ["delete", "--target", target, "--slug", "eval", "--yes"] + (["--dry-run"] if dry_run else [])
+ assert incus_regression.main(flags) == 1
+ captured = capsys.readouterr()
+ assert "session-one" in captured.err and "Evaluation in progress" in captured.err
+ assert "--ignore-hold" in captured.err
+ assert "+ incus" not in captured.out
+ assert commands == []
+ assert config == original
+ if target == "worktree":
+ assert not incus_regression.mapping_path(tmp_path).exists()
+ flags.append("--ignore-hold")
+ assert incus_regression.main(flags) == 0
+ project, instance = ("avr-master", "avibe-master") if target == "master" else ("avr-wt-eval", "avibe-wt-eval")
+ expected = [
+ ["incus", "--project", project, "delete", instance, "--force"],
+ ["incus", "project", "delete", project],
+ ]
+ if dry_run:
+ out = capsys.readouterr().out
+ assert all("+ " + shlex.join(command) in out for command in expected)
+ assert commands == []
+ assert config == original
+ else:
+ assert commands == expected
+
+
[email protected](("dry_run", "ignore_hold"), [(False, False), (True, False), (True, True)])
+def test_delete_requires_incus_for_live_reads_as_well_as_writes(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], dry_run: bool, ignore_hold: bool
+) -> None:
+ """A dry-run live read must validate its executable; only an overridden preview is fully offline."""
+ monkeypatch.setattr(incus_regression, "current_repo_root", lambda: tmp_path)
+ monkeypatch.setattr(incus_regression, "git_common_root", lambda root: root)
+ monkeypatch.setattr(incus_regression, "load_env_file", lambda *args: None)
+ monkeypatch.setenv("INCUS_CMD", "/missing/incus")
+ monkeypatch.setattr(incus_regression.shutil, "which", lambda executable: None)
+ attempted = []
+
+ def missing_executable(command, **kwargs):
+ attempted.append(command)
+ raise FileNotFoundError(2, "No such file or directory", command[0])
+
+ monkeypatch.setattr(incus_regression.subprocess, "run", missing_executable)
+ flags = ["delete", "--target", "worktree", "--slug", "eval", "--yes"]
+ if dry_run:
+ flags.append("--dry-run")
+ if ignore_hold:
+ flags.append("--ignore-hold")
+ assert incus_regression.main(flags) == (0 if dry_run and ignore_hold else 1)
+ assert attempted == []
+ err = capsys.readouterr().err
+ if not (dry_run and ignore_hold):
+ assert "The Incus CLI executable was not found: /missing/incus" in err
+
+
+def test_reconcile_reports_holds_on_every_observed_instance_without_an_override(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+ """Untracked and misplaced instances must retain their holds in the inventory report."""
+ mapping_path, commands = reconcile_fixture(
+ tmp_path, monkeypatch, entries={}, projects=("avr-wt-eval",),
+ instances=tuple(
+ {
+ "name": "avibe-wt-eval", "project": project, "status": "Running",
+ "config": {
+ "user.avibe-regression.hold.owner": owner,
+ "user.avibe-regression.hold.reason": "Long evaluation",
+ "user.avibe-regression.hold.since": "2026-10-20T12:00:00+00:00",
+ },
+ }
+ for project, owner in (("avr-wt-eval", "session-one"), ("default", "session-two"))
+ ),
+ )
+ before = mapping_path.read_bytes()
+ assert incus_regression.main(["reconcile", "--yes"]) == 0
+ out = capsys.readouterr().out
+ for project, owner in (("avr-wt-eval", "session-one"), ("default", "session-two")):
+ assert (
+ f"{project}/avibe-wt-eval: HELD: owner={owner}; "
+ "reason=Long evaluation; since=2026-10-20T12:00:00+00:00"
+ ) in out
+ assert "delete --target worktree --slug eval --yes" in out
+ assert "--ignore-hold" not in out
+ assert not any("delete" in command for command in commands)
+ assert mapping_path.read_bytes() == before
+
+
def test_reconcile_reports_environments_incus_holds_without_metadata(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
@@ -3261,6 +3429,7 @@ def test_delete_round_trips_generated_worktree_slug(tmp_path: Path, monkeypatch:
dry_run=False,
remote=None,
yes=True,
+ ignore_hold=False,
)
)
@@ -3270,6 +3439,7 @@ def test_delete_round_trips_generated_worktree_slug(tmp_path: Path, monkeypatch:
["incus", "project", "delete", project],
]
assert listings == [
+ (["incus", "project", "list", "--format", "json"], "Incus projects"),
(["incus", "project", "list", "--format", "json"], "Incus projects"),
(["incus", "list", "--all-projects", "--format", "json"], "Incus instances"),
]
@@ -3311,9 +3481,18 @@ def test_delete_keeps_worktree_metadata_until_absence_is_confirmed(
["incus", "list", "--all-projects", "--format", "json"],
]
commands = []
+ preflight = [
+ listing_commands[0],
+ ["incus", "--project", row["project"], "list", "--format", "json"],
+ ]
def run(command, **kwargs):
assert incus_regression.target_run_in_flight(repo, None, row["project"])
+ if len(commands) < len(preflight):
+ assert command == preflight[len(commands)]
+ objects = [{"name": row["project"]}] if not commands else [{"name": row["instance"], "config": {}}]
+ commands.append(command)
+ return subprocess.CompletedProcess(command, 0, stdout=json.dumps(objects))
commands.append(command)
if command in delete_commands:
assert kwargs["check"] is False
@@ -3349,7 +3528,7 @@ def test_delete_keeps_worktree_metadata_until_absence_is_confirmed(
listing_commands[:1] if inventory in {"project-list-fails", "unreadable-project-list"}
else listing_commands
)
- assert commands == delete_commands + expected_listings
+ assert commands == preflight + delete_commands + expected_listings
assert not incus_regression.target_run_in_flight(repo, None, row["project"])
payload = json.loads(mapping_path.read_text(encoding="utf-8"))
assert payload["worktrees"] == ({"another-worktree": other_row} if inventory == "absent" else rows)
@@ -3380,6 +3559,8 @@ def test_delete_against_a_remote_keeps_the_local_metadata(
mapping_path.write_text(json.dumps({"schema_version": 1, "worktrees": {slug: row}}), encoding="utf-8")
class RecordingRunner:
+ names, records = daemon_listing(row["project"], row["instance"])
+
def __init__(self, *, dry_run=False):
self.dry_run = dry_run
@@ -3405,6 +3586,7 @@ def test_delete_against_a_remote_keeps_the_local_metadata(
dry_run=False,
remote="lab",
yes=True,
+ ignore_hold=False,
)
)
@@ -3427,6 +3609,7 @@ def _delete_args(slug: str, **overrides) -> argparse.Namespace:
"dry_run": False,
"remote": None,
"yes": True,
+ "ignore_hold": False,
}
fields.update(overrides)
return argparse.Namespace(**fields)
@@ -5095,9 +5278,10 @@ def test_an_empty_remote_names_the_local_daemon_for_every_command() -> None:
assert checked, "no subcommand accepts --remote"
for name in checked:
+ required = ["--owner", "session", "--reason", "evaluation"] if name == "hold" else []
for spelling in ("", " "):
- assert parser.parse_args([name, "--remote", spelling]).remote is None, name
- assert parser.parse_args([name, "--remote", " lab "]).remote == "lab", name
+ assert parser.parse_args([name, *required, "--remote", spelling]).remote is None, name
+ assert parser.parse_args([name, *required, "--remote", " lab "]).remote == "lab", name
# The consequence, at the reader that used to disagree with `remote_ref`.
empty = parser.parse_args(["up", "--remote", ""])