Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(hogli): add a build mode to worktrees:clean (#130052)

Robbieea70b5d84af8
diff --git a/docs/published/handbook/engineering/flox-multi-instance-workflow.md b/docs/published/handbook/engineering/flox-multi-instance-workflow.md
index 98a18ca56663..5c21bff433e6 100644
--- a/docs/published/handbook/engineering/flox-multi-instance-workflow.md
+++ b/docs/published/handbook/engineering/flox-multi-instance-workflow.md
@@ -402,11 +402,18 @@ phw remove haacked/old-feature
git worktree prune
```
-To clear old `phw`, Claude Code, Codex, and PostHog Code worktrees in bulk, run `hogli worktrees:clean --before 7d --mode deps --dry-run` to preview, then drop `--dry-run`, or use `--mode full` to remove whole worktrees.
-Both modes skip a worktree where `cargo` or `rustc` is running.
-`--mode deps` keeps a Cargo `target` that a build used after the cutoff, and, without `--source` or `--repo`, offers to delete the old shared `~/.cargo/target` when `CARGO_TARGET_DIR` is not set and no build used it after the cutoff.
-
-Flox activation also runs this once a day in the background: `--mode deps` on worktrees idle for 7 days and `--mode full` on worktrees idle for 14 days.
+To clear old `phw`, Claude Code (including agent scratchpads), Codex, and PostHog Code worktrees in bulk, run `hogli worktrees:clean --before 7d --mode deps,build --dry-run` to preview, then drop `--dry-run`, or use `--mode full` to remove whole worktrees.
+`--mode deps` removes installed dependencies such as `node_modules`, Python venvs, and Cargo `target` dirs.
+`--mode build` removes build output and tool caches such as Cargo `target`, `dist`, `.next`, and `__pycache__`, but only where git ignores every file inside.
+Neither removes a folder that holds a tracked file.
+Every mode skips a worktree where `cargo` or `rustc` is running.
+`--mode deps` and `--mode build` keep a Cargo `target` that a build used after the cutoff.
+Without `--source` or `--repo`, they also offer to delete the old shared `~/.cargo/target` when `CARGO_TARGET_DIR` is not set and no build used it after the cutoff.
+For a Cargo target that several projects share, run `hogli doctor:disk --area cargo-units`. It removes only the build units that no build in the last 7 days needed, directly or through its dependencies.
+Every mode also removes folders of removed worktrees that hold only flox caches.
+`--mode deps` and `--mode build` also act on worktrees that `git worktree list` registers outside these folders, such as ones made by hand. `--mode full` and the daily background run act on those only with `--source other`.
+
+Flox activation also runs this once a day in the background: `--mode deps,build` on worktrees idle for 7 days and `--mode full` on worktrees idle for 14 days.
It skips worktrees with uncommitted or unpushed work and keeps every branch.
Set `POSTHOG_WORKTREES_AUTOCLEAN=0` to turn it off.
The output goes to `.flox/cache/worktrees-clean.log` in the checkout that ran it.
diff --git a/tools/hogli-commands/hogli_commands/cargo_targets.py b/tools/hogli-commands/hogli_commands/cargo_targets.py
index cfd88c140c4f..0c9e7e995645 100644
--- a/tools/hogli-commands/hogli_commands/cargo_targets.py
+++ b/tools/hogli-commands/hogli_commands/cargo_targets.py
@@ -1,10 +1,16 @@
from __future__ import annotations
import os
+import re
+import json
+import fcntl
+import tomllib
+import contextlib
import subprocess
-from collections.abc import Sequence
+from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from pathlib import Path
+from typing import IO
from .processes import process_cwds
@@ -27,7 +33,9 @@ def inode_key(path: Path) -> InodeKey | None:
def _cargo_pids() -> list[int] | None:
try:
- result = subprocess.run(["pgrep", "-x", "cargo|rustc"], capture_output=True, text=True, timeout=5, check=False)
+ result = subprocess.run(
+ ["pgrep", "-x", "cargo|rustc|cargo-nextest"], capture_output=True, text=True, timeout=5, check=False
+ )
except (OSError, subprocess.SubprocessError):
return None
if result.returncode not in (0, 1):
@@ -96,13 +104,249 @@ def last_cargo_build_epoch(target: Path) -> float:
return newest
+def looks_like_cargo_target(path: Path) -> bool:
+ # Cargo writes CACHEDIR.TAG into every target dir it creates, and other build tools also make `target/release`.
+ return (path / "CACHEDIR.TAG").is_file()
+
+
+@dataclass(frozen=True, kw_only=True)
+class SharedCargoTarget:
+ path: Path
+ reason: str
+
+
+def shared_cargo_targets() -> list[SharedCargoTarget]:
+ """Cargo target dirs that live outside any checkout, so builds from every repo and worktree go into them."""
+
+ # A target inside a checkout is that checkout's own, and only a caller that checks git may remove it.
+ return [shared for shared in configured_cargo_targets() if not _inside_git_work_tree(shared.path)]
+
+
+def configured_cargo_targets() -> list[SharedCargoTarget]:
+ """The target dirs that CARGO_TARGET_DIR, build.target-dir in the Cargo config, or the old Flox env point Cargo at."""
+
+ cargo_home = Path(os.environ.get("CARGO_HOME") or Path.home() / ".cargo")
+ candidates: list[SharedCargoTarget] = []
+ if env_target := os.environ.get("CARGO_TARGET_DIR"):
+ candidates.append(SharedCargoTarget(path=Path(env_target), reason="CARGO_TARGET_DIR points here."))
+ else:
+ candidates.append(
+ SharedCargoTarget(
+ path=Path.home() / ".cargo" / "target",
+ reason="Cargo built here while the Flox env set CARGO_TARGET_DIR.",
+ )
+ )
+ if config_target := _configured_target_dir(cargo_home / "config.toml"):
+ candidates.append(
+ SharedCargoTarget(
+ path=config_target,
+ reason=f"build.target-dir in {cargo_home / 'config.toml'} points here.",
+ )
+ )
+
+ unique: dict[InodeKey, SharedCargoTarget] = {}
+ for candidate in candidates:
+ if candidate.path.is_absolute() and (key := inode_key(candidate.path)) is not None:
+ unique.setdefault(key, candidate)
+ return list(unique.values())
+
+
+def _inside_git_work_tree(path: Path) -> bool:
+ """Whether *path* or a folder above it holds a `.git` entry, which also catches a checkout whose git metadata is broken."""
+
+ return any((directory / ".git").exists() for directory in (path, *path.parents))
+
+
+def _configured_target_dir(config: Path) -> Path | None:
+ try:
+ target_dir = tomllib.loads(config.read_text()).get("build", {}).get("target-dir")
+ except (OSError, ValueError, AttributeError):
+ return None
+ return Path(target_dir) if isinstance(target_dir, str) else None
+
+
+def unused_cargo_target(target: Path, built_before_epoch: float) -> bool:
+ """Whether no build used *target* after *built_before_epoch* and no cargo runs that could be using it now."""
+
+ return last_cargo_build_epoch(target) < built_before_epoch and idle_cargo_target(target)
+
+
+def idle_cargo_target(target: Path) -> bool:
+ """Whether *target* is a real Cargo target dir and no cargo or rustc runs that could be writing to it."""
+
+ if target.is_symlink() or not looks_like_cargo_target(target):
+ return False
+ # A shell with another environment can build here, and its CARGO_TARGET_DIR is not visible from this process.
+ return _cargo_pids() == []
+
+
def unused_legacy_cargo_target(built_before_epoch: float) -> Path | None:
"""The ~/.cargo/target that Cargo used when the Flox env set CARGO_TARGET_DIR, if no build used it after *built_before_epoch*."""
target = Path.home() / ".cargo" / "target"
- if os.environ.get("CARGO_TARGET_DIR") or not target.is_dir() or target.is_symlink():
+ if os.environ.get("CARGO_TARGET_DIR"):
+ return None
+ return target if unused_cargo_target(target, built_before_epoch) else None
+
+
+_METADATA_HASH = re.compile(r"-([0-9a-f]{16})(?=\.|$)")
+_UNIT_OUTPUT_DIRS = ("deps", "build", "examples")
+
+
+@dataclass(frozen=True, kw_only=True)
+class _CargoUnit:
+ fingerprint_dir: Path
+ built_epoch: float
+ dependency_fingerprints: list[str]
+
+
+def unused_cargo_units(target: Path, built_before_epoch: float) -> list[Path] | None:
+ """The paths of the units in *target* that no unit compiled after *built_before_epoch* needs, and the incremental caches not written since then.
+
+ Cargo does not touch the fingerprint of a unit it reuses, and the file
+ systems this runs on do not record reads, so a unit's own age cannot show
+ that a build still uses it. Its dependents can: every unit's fingerprint
+ JSON names the fingerprints of the units it depends on. A unit stays when
+ any unit compiled in the window reaches it through that graph. A build in
+ which every unit was fresh compiles nothing, so a profile that Cargo
+ locked in the window without compiling a unit there is kept whole. Deleting
+ a unit only costs a rebuild, because Cargo rebuilds a unit whose files are
+ missing. Returns None, and nothing should be removed, when a fingerprint
+ cannot be read, because a dependency edge could be missing.
+ """
+
+ try:
+ target = target.resolve(strict=True)
+ except (OSError, RuntimeError):
return None
- if last_cargo_build_epoch(target) >= built_before_epoch:
+ units: dict[Path, _CargoUnit] = {}
+ unit_by_fingerprint: dict[str, Path] = {}
+ profiles = _cargo_profiles(target)
+ unit_dirs_by_profile: dict[Path, list[Path]] = {}
+ try:
+ for profile in profiles:
+ unit_dirs = [
+ entry for entry in (profile / ".fingerprint").iterdir() if entry.is_dir() and not entry.is_symlink()
+ ]
+ unit_dirs_by_profile[profile] = unit_dirs
+ for unit_dir in unit_dirs:
+ unit = _read_cargo_unit(unit_dir, unit_by_fingerprint)
+ if unit is None:
+ return None
+ units[unit_dir] = unit
+ except OSError:
+ return None
+
+ live: set[Path] = set()
+ pending = [unit_dir for unit_dir, unit in units.items() if unit.built_epoch >= built_before_epoch]
+ while pending:
+ unit_dir = pending.pop()
+ if unit_dir in live:
+ continue
+ live.add(unit_dir)
+ for fingerprint in units[unit_dir].dependency_fingerprints:
+ if (dependency := unit_by_fingerprint.get(fingerprint)) is not None:
+ pending.append(dependency)
+
+ unused: list[Path] = []
+ for profile, unit_dirs in unit_dirs_by_profile.items():
+ locked_without_compiling = _mtime(profile / ".cargo-lock") >= built_before_epoch and not any(
+ units[unit_dir].built_epoch >= built_before_epoch for unit_dir in unit_dirs
+ )
+ if locked_without_compiling:
+ continue
+ outputs_by_hash: dict[str, list[Path]] = {}
+ for output_dir in _UNIT_OUTPUT_DIRS:
+ try:
+ entries = list((profile / output_dir).iterdir())
+ except OSError:
+ continue
+ for entry in entries:
+ if match := _METADATA_HASH.search(entry.name):
+ outputs_by_hash.setdefault(match.group(1), []).append(entry)
+ for unit_dir in unit_dirs:
+ if unit_dir not in live:
+ unused.append(unit_dir)
+ unused.extend(outputs_by_hash.get(unit_dir.name.rpartition("-")[2], []))
+ try:
+ incremental = [entry for entry in (profile / "incremental").iterdir() if entry.is_dir()]
+ except OSError:
+ incremental = []
+ # An incremental cache only speeds up recompiling its crate, and a compile writes a new session into it.
+ unused.extend(entry for entry in incremental if _newest_child_mtime(entry) < built_before_epoch)
+ # A symlinked folder inside the target, such as an incremental dir, would make a removal follow it out of the target.
+ return [
+ path for path in unused if os.path.realpath(path.parent) == str(path.parent) and path.is_relative_to(target)
+ ]
+
+
+def _cargo_profiles(target: Path) -> list[Path]:
+ # Tools such as rust-analyzer and cargo-llvm-cov build into <target>/<tool>/<triple>/<profile>.
+ return [
+ fingerprints.parent
+ for pattern in ("*/.fingerprint", "*/*/.fingerprint", "*/*/*/.fingerprint")
+ for fingerprints in target.glob(pattern)
+ ]
+
+
+def cargo_target_locked(target: Path) -> Iterator[bool]:
+ """Hold Cargo's own build lock on every profile of *target* until the block ends, or yield False when a build holds one.
+
+ Cargo takes an exclusive flock on <profile>/.cargo-lock for every build,
+ so while this holds them, a build that starts waits instead of reading
+ files that the caller is removing.
+ """
+
+ held: list[IO[str]] = []
+ try:
+ for profile in _cargo_profiles(target):
+ lock = (profile / ".cargo-lock").open("a")
+ try:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError:
+ lock.close()
+ yield False
+ return
+ held.append(lock)
+ yield True
+ finally:
+ for held_lock in held:
+ held_lock.close()
+
+
+def _mtime(path: Path) -> float:
+ try:
+ return path.stat().st_mtime
+ except OSError:
+ return 0.0
+
+
+def _read_cargo_unit(unit_dir: Path, unit_by_fingerprint: dict[str, Path]) -> _CargoUnit | None:
+ dependency_fingerprints: list[str] = []
+ try:
+ for entry in unit_dir.iterdir():
+ if entry.suffix == ".json":
+ for dependency in json.loads(entry.read_text())["deps"]:
+ # Cargo stores each fingerprint as a u64, written little-endian in hex in the unit's hash file.
+ dependency_fingerprints.append(int(dependency[3]).to_bytes(8, "little").hex())
+ elif entry.is_file() and not entry.name.startswith(("dep-", "output-", "invoked")):
+ unit_by_fingerprint[entry.read_text().strip()] = unit_dir
+ except (OSError, ValueError, KeyError, IndexError, TypeError, OverflowError):
return None
- # A shell with an older Flox env can still build here, and its CARGO_TARGET_DIR is not visible from this process.
- return target if _cargo_pids() == [] else None
+ return _CargoUnit(
+ fingerprint_dir=unit_dir,
+ built_epoch=_newest_child_mtime(unit_dir),
+ dependency_fingerprints=dependency_fingerprints,
+ )
+
+
+def _newest_child_mtime(directory: Path) -> float:
+ newest = 0.0
+ try:
+ with os.scandir(directory) as entries:
+ for entry in entries:
+ newest = max(newest, entry.stat(follow_symlinks=False).st_mtime)
+ except OSError:
+ return float("inf")
+ return newest
diff --git a/tools/hogli-commands/hogli_commands/disk_usage.py b/tools/hogli-commands/hogli_commands/disk_usage.py
new file mode 100644
index 000000000000..56d6e93392ef
--- /dev/null
+++ b/tools/hogli-commands/hogli_commands/disk_usage.py
@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import os
+import sys
+import stat
+import ctypes
+import struct
+from collections.abc import Callable, Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class DiskUsage:
+ freeable_bytes: int
+ allocated_bytes: int
+
+ def __add__(self, other: DiskUsage) -> DiskUsage:
+ return DiskUsage(
+ freeable_bytes=self.freeable_bytes + other.freeable_bytes,
+ allocated_bytes=self.allocated_bytes + other.allocated_bytes,
+ )
+
+
+NO_USAGE = DiskUsage(freeable_bytes=0, allocated_bytes=0)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class _FileBlocks:
+ inode: int
+ link_count: int
+ allocated_bytes: int
+ unshared_bytes: int
+
+
+def measure_disk_usage(paths: Sequence[Path]) -> DiskUsage:
+ """What deleting *paths* would free, next to what they occupy.
+
+ A file whose blocks are shared frees nothing while the other owner keeps
+ them. pnpm and uv clone each file from their store on APFS and hard-link
+ it on other file systems, so `du` reports a dependency folder as many
+ times larger than its deletion frees. A hard-linked file counts as
+ freeable only when every one of its links is inside *paths*. Symlinks and
+ other mounted file systems are not followed.
+ """
+
+ # One int per hard-linked inode, because pnpm hard-links every file in a large tree on Linux.
+ links_not_seen: dict[tuple[int, int], int] = {}
+ freeable = allocated = 0
+ for path in paths:
+ for device, blocks in _walk_files(path):
+ if blocks.link_count <= 1:
+ allocated += blocks.allocated_bytes
+ freeable += blocks.unshared_bytes
+ continue
+ key = (device, blocks.inode)
+ remaining = links_not_seen.get(key)
+ if remaining is None:
+ allocated += blocks.allocated_bytes
+ remaining = blocks.link_count
+ remaining -= 1
+ if remaining == 0:
+ links_not_seen.pop(key, None)
+ freeable += blocks.unshared_bytes
+ else:
+ links_not_seen[key] = remaining
+ return DiskUsage(freeable_bytes=freeable, allocated_bytes=allocated)
+
+
+def _walk_files(path: Path) -> Iterator[tuple[int, _FileBlocks]]:
+ try:
+ device = path.lstat().st_dev
+ if path.is_symlink() or not path.is_dir():
+ if path.is_file() and (blocks := _file_blocks(str(path))) is not None:
+ yield device, blocks
+ return
+ except OSError:
+ return
+ pending = [str(path)]
+ while pending:
+ directory = pending.pop()
+ try:
+ with os.scandir(directory) as entries:
+ for entry in entries:
+ if entry.is_dir(follow_symlinks=False):
+ # Another file system mounted inside frees nothing here, and a stale network mount can block the walk.
+ if entry.stat(follow_symlinks=False).st_dev == device:
+ pending.append(entry.path)
+ elif entry.is_file(follow_symlinks=False) and (blocks := _file_blocks(entry.path)) is not None:
+ yield device, blocks
+ except OSError:
+ continue
+
+
+def _blocks_from_lstat(path: str) -> _FileBlocks | None:
+ try:
+ status = os.lstat(path)
+ except OSError:
+ return None
+ if not stat.S_ISREG(status.st_mode):
+ return None
+ allocated = status.st_blocks * 512
+ return _FileBlocks(
+ inode=status.st_ino, link_count=status.st_nlink, allocated_bytes=allocated, unshared_bytes=allocated
+ )
+
+
+def _apfs_blocks_reader() -> Callable[[str], _FileBlocks | None]:
+ """Read a file's blocks with getattrlist(2), whose ATTR_CMNEXT_PRIVATESIZE excludes the blocks that APFS clones share."""
+
+ libc = ctypes.CDLL(None, use_errno=True)
+ getattrlist = libc.getattrlist
+ getattrlist.argtypes = [ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32]
+ getattrlist.restype = ctypes.c_int
+
+ class AttrList(ctypes.Structure):
+ _fields_ = [
+ ("bitmapcount", ctypes.c_ushort),
+ ("reserved", ctypes.c_uint16),
+ ("commonattr", ctypes.c_uint32),
+ ("volattr", ctypes.c_uint32),
+ ("dirattr", ctypes.c_uint32),
+ ("fileattr", ctypes.c_uint32),
+ ("forkattr", ctypes.c_uint32),
+ ]
+
+ attr_cmn_fileid, attr_file_linkcount, attr_file_allocsize, attr_cmnext_privatesize = 0x02000000, 0x1, 0x4, 0x8
+ fsopt_nofollow, fsopt_attr_cmn_extended = 0x1, 0x20
+ request = AttrList(5, 0, attr_cmn_fileid, 0, 0, attr_file_linkcount | attr_file_allocsize, attr_cmnext_privatesize)
+ options = fsopt_nofollow | fsopt_attr_cmn_extended
+ # The kernel packs the requested attributes after a u32 length, in group order and then bit order within a group.
+ layout = struct.Struct("<IQIqq")
+
+ def read(path: str) -> _FileBlocks | None:
+ buffer = ctypes.create_string_buffer(layout.size)
+ if getattrlist(os.fsencode(path), ctypes.byref(request), buffer, layout.size, options) != 0:
+ return _blocks_from_lstat(path)
+ returned_length, inode, link_count, allocated, unshared = layout.unpack(buffer.raw)
+ if returned_length != layout.size:
+ return _blocks_from_lstat(path)
+ return _FileBlocks(inode=inode, link_count=link_count, allocated_bytes=allocated, unshared_bytes=unshared)
+
+ return read
+
+
+_file_blocks: Callable[[str], _FileBlocks | None] = (
+ _apfs_blocks_reader() if sys.platform == "darwin" else _blocks_from_lstat
+)
diff --git a/tools/hogli-commands/hogli_commands/doctor.py b/tools/hogli-commands/hogli_commands/doctor.py
index 78de0928a946..04deb76138b0 100644
--- a/tools/hogli-commands/hogli_commands/doctor.py
+++ b/tools/hogli-commands/hogli_commands/doctor.py
@@ -27,7 +27,16 @@ import click
from hogli.manifest import REPO_ROOT, get_manifest
from . import hints
-from .cargo_targets import unused_legacy_cargo_target
+from .cargo_targets import (
+ cargo_target_locked,
+ configured_cargo_targets,
+ idle_cargo_target,
+ looks_like_cargo_target,
+ shared_cargo_targets,
+ unused_cargo_units,
+ unused_legacy_cargo_target,
+)
+from .disk_usage import measure_disk_usage
from .processes import process_cwds
MAX_SAMPLE_PATHS = 8
@@ -146,6 +155,7 @@ class CleanupResult:
"staticfiles",
"dagster",
"node-artifacts",
+ "cargo-units",
"rust",
"legacy-cargo-target",
"sccache",
@@ -175,6 +185,7 @@ def doctor_disk(
- Dagster background job storage
- pnpm/Vite/Tailwind/Storybook/Playwright build artifacts
- Rust workspaces built with Cargo, plus the sccache compilation cache
+ - Cargo build units in shared and local target dirs that no recent build needs
- The old shared Cargo target at ~/.cargo/target
- The uv and pnpm package caches shared across every worktree
@@ -281,6 +292,20 @@ def doctor_disk(
cleanup=_cleanup_items,
confirmation_prompt="Clean up JavaScript build caches and artifacts?",
),
+ CleanupCategory(
+ id="cargo_units",
+ title="🦀 Cargo build units that no recent build needs",
+ description=[
+ f"Removes the units of a Cargo target dir that no unit compiled in the last {STALE_RUST_TARGET_DAYS} days needs,",
+ "directly or through other units, and incremental caches not written since then.",
+ "Covers CARGO_TARGET_DIR, build.target-dir in ~/.cargo/config.toml, ~/.cargo/target and this checkout.",
+ "Cargo writes nothing when a build finds every unit fresh, so a project that only ran such",
+ "builds in that time looks unused, and its next build recompiles what was removed.",
+ ],
+ estimate=_estimate_unused_cargo_units,
+ cleanup=_cleanup_unused_cargo_units,
+ confirmation_prompt="Remove the Cargo build units that no recent build needs?",
+ ),
CleanupCategory(
id="rust",
title="🦀 Rust Cargo targets",
@@ -1429,6 +1454,104 @@ def _collect_rust_target_dirs(repo_root: Path) -> list[CleanupItem]:
return items
+def _sweepable_cargo_targets(repo_root: Path) -> list[Path]:
+ """Resolved Cargo targets outside every checkout, plus configured and local targets that their checkout's git ignores."""
+
+ outside_checkouts = {shared.path for shared in shared_cargo_targets()}
+ candidates = list(outside_checkouts)
+ candidates += [
+ configured.path for configured in configured_cargo_targets() if configured.path not in outside_checkouts
+ ]
+ try:
+ listed = subprocess.run(
+ ["git", "ls-files", "-z", "--", "Cargo.toml", ":(glob)**/Cargo.toml"],
+ cwd=repo_root,
+ capture_output=True,
+ text=True,
+ errors="surrogateescape",
+ timeout=60,
+ check=False,
+ )
+ manifests = listed.stdout.split("\0") if listed.returncode == 0 else []
+ except (OSError, subprocess.SubprocessError):
+ manifests = []
+ candidates += [(repo_root / manifest).parent / "target" for manifest in manifests if manifest]
+
+ targets: dict[Path, None] = {}
+ for target in candidates:
+ try:
+ resolved = target.resolve(strict=True)
+ except (OSError, RuntimeError):
+ continue
+ if not looks_like_cargo_target(resolved):
+ continue
+ if target in outside_checkouts or _ignored_by_its_checkout(resolved):
+ targets.setdefault(resolved, None)
+ return list(targets)
+
+
+def _ignored_by_its_checkout(path: Path) -> bool:
+ try:
+ result = subprocess.run(
+ ["git", "check-ignore", "-q", "--", path.name],
+ cwd=path.parent,
+ capture_output=True,
+ timeout=30,
+ check=False,
+ )
+ except (OSError, subprocess.SubprocessError):
+ return False
+ return result.returncode == 0
+
+
+def _unused_units_by_target(repo_root: Path) -> tuple[dict[Path, list[Path]], list[str]]:
+ built_before_epoch = time.time() - STALE_RUST_TARGET_DAYS * 24 * 60 * 60
+ unused_by_target: dict[Path, list[Path]] = {}
+ skipped: list[str] = []
+ for target in _sweepable_cargo_targets(repo_root):
+ if not idle_cargo_target(target):
+ skipped.append(f" {target}: skipped, because a cargo, rustc or cargo-nextest process may be using it.")
+ continue
+ unused = unused_cargo_units(target, built_before_epoch)
+ if unused is None:
+ skipped.append(f" {target}: skipped, because a Cargo fingerprint could not be read.")
+ elif unused:
+ unused_by_target[target] = unused
+ return unused_by_target, skipped
+
+
+def _estimate_unused_cargo_units(repo_root: Path) -> CleanupEstimate:
+ unused_by_target, details = _unused_units_by_target(repo_root)
+ items: list[CleanupItem] = []
+ for target, unused in unused_by_target.items():
+ target_items = [
+ CleanupItem(path, float(measure_disk_usage([path]).freeable_bytes), is_dir=path.is_dir()) for path in unused
+ ]
+ details.append(
+ f" {target}: {len(target_items)} unit file(s) and cache(s), "
+ f"{_format_size(sum(item.size for item in target_items))}"
+ )
+ items.extend(target_items)
+ if not items and not details:
+ details.append(" Nothing to remove.")
+ return CleanupEstimate(total_size=sum(item.size for item in items), items=items, details=details)
+
+
+def _cleanup_unused_cargo_units(estimate: CleanupEstimate, repo_root: Path) -> CleanupStats:
+ # Holding Cargo's build lock makes a build that starts now wait, and the rescan under it drops units a build used during the prompt.
+ estimated = {item.path: item for item in estimate.items}
+ built_before_epoch = time.time() - STALE_RUST_TARGET_DAYS * 24 * 60 * 60
+ freed = 0.0
+ for target in _sweepable_cargo_targets(repo_root):
+ with cargo_target_locked(target) as locked:
+ if not locked or not idle_cargo_target(target):
+ click.echo(f" {target}: a build is running, so it is kept.")
+ continue
+ unused = unused_cargo_units(target, built_before_epoch) or []
+ freed += _delete_items([estimated[path] for path in unused if path in estimated])
+ return CleanupStats(freed=freed, deleted_anything=freed > 0)
+
+
def _estimate_legacy_cargo_target(_repo_root: Path) -> CleanupEstimate:
pointer = f" For the rust/target of other worktrees, run: hogli worktrees:clean --before {STALE_RUST_TARGET_DAYS}d --mode deps"
target = unused_legacy_cargo_target(time.time() - STALE_RUST_TARGET_DAYS * 24 * 60 * 60)
diff --git a/tools/hogli-commands/hogli_commands/prechecks.py b/tools/hogli-commands/hogli_commands/prechecks.py
index 569b2711ec63..4c136b31d328 100644
--- a/tools/hogli-commands/hogli_commands/prechecks.py
+++ b/tools/hogli-commands/hogli_commands/prechecks.py
@@ -24,7 +24,7 @@ _DEFAULT_MIN_FREE_GIB = 20.0
_DEFAULT_MAX_WORKTREES = 3
_MIN_FREE_GIB_ENV = "HOGLI_WORKTREE_DISK_MIN_FREE_GIB"
_MAX_WORKTREES_ENV = "HOGLI_WORKTREE_DISK_MAX_WORKTREES"
-_CLEAN_PREVIEW_COMMAND = "hogli worktrees:clean --before 7d --mode deps --dry-run"
+_CLEAN_PREVIEW_COMMAND = "hogli worktrees:clean --before 7d --mode deps,build --dry-run"
def _migrations_precheck(check: dict, yes: bool) -> bool | None:
diff --git a/tools/hogli-commands/hogli_commands/tests/test_disk_usage.py b/tools/hogli-commands/hogli_commands/tests/test_disk_usage.py
new file mode 100644
index 000000000000..73d1d44811e3
--- /dev/null
+++ b/tools/hogli-commands/hogli_commands/tests/test_disk_usage.py
@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import os
+import sys
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from hogli_commands.disk_usage import measure_disk_usage
+
+_FILE_BYTES = 256 * 1024
+
+
+def _write_file(path: Path) -> Path:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(os.urandom(_FILE_BYTES))
+ return path
+
+
+def _clone(source: Path, destination: Path) -> None:
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ subprocess.run(["cp", "-c", str(source), str(destination)], check=True)
+
+
+ ("layout", "expected_freeable_files", "expected_allocated_files"),
+ [
+ ("own_file", 1, 1),
+ ("hardlink_from_outside", 0, 1),
+ ("both_hardlinks_inside", 1, 1),
+ ("symlink_to_outside", 0, 0),
+ pytest.param(
+ "apfs_clone_from_outside",
+ 0,
+ 1,
+ marks=pytest.mark.skipif(sys.platform != "darwin", reason="APFS clones need macOS"),
+ ),
+ ],
+)
+def test_counts_as_freeable_only_blocks_no_file_outside_the_paths_shares(
+ tmp_path, layout: str, expected_freeable_files: int, expected_allocated_files: int
+) -> None:
+ store = _write_file(tmp_path / "store" / "package.js")
+ deps = tmp_path / "wt" / "node_modules"
+ deps.mkdir(parents=True)
+ if layout == "own_file":
+ _write_file(deps / "package.js")
+ elif layout == "hardlink_from_outside":
+ os.link(store, deps / "package.js")
+ elif layout == "both_hardlinks_inside":
+ _write_file(deps / "a.js")
+ os.link(deps / "a.js", deps / "b.js")
+ elif layout == "symlink_to_outside":
+ (deps / "package.js").symlink_to(store)
+ else:
+ _clone(store, deps / "package.js")
+
+ usage = measure_disk_usage([deps])
+
+ assert usage.freeable_bytes == pytest.approx(expected_freeable_files * _FILE_BYTES, rel=0.1)
+ assert usage.allocated_bytes == pytest.approx(expected_allocated_files * _FILE_BYTES, rel=0.1)
diff --git a/tools/hogli-commands/hogli_commands/tests/test_doctor.py b/tools/hogli-commands/hogli_commands/tests/test_doctor.py
index 98536562109e..11495749db61 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_doctor.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_doctor.py
@@ -55,6 +55,7 @@ from hogli_commands.doctor import (
_scan_port_holders,
_scan_unheld_via_lsof,
_select_flox_logs_to_remove,
+ _sweepable_cargo_targets,
_tail,
doctor_git,
doctor_migrate_volumes,
@@ -2010,6 +2011,24 @@ def test_estimate_sccache_accepts_a_directory_of_its_own(monkeypatch: pytest.Mon
assert estimate.total_size == 2048
[email protected]("ignored", [True, False], ids=["ignored_by_its_checkout", "not_ignored"])
+def test_cargo_units_area_sweeps_a_target_inside_a_checkout_only_when_git_ignores_it(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ignored: bool
+) -> None:
+ checkout = tmp_path / "checkout"
+ checkout.mkdir()
+ subprocess.run(["git", "init", "-q"], cwd=checkout, check=True)
+ if ignored:
+ (checkout / ".gitignore").write_text(".build-cache/\n")
+ target = checkout / ".build-cache"
+ (target / "debug" / ".fingerprint").mkdir(parents=True)
+ (target / "CACHEDIR.TAG").write_text("Signature: 8a477f597d28d172789f06886806bc55\n")
+ monkeypatch.setattr(Path, "home", lambda: tmp_path / "home")
+ monkeypatch.setenv("CARGO_TARGET_DIR", str(target))
+
+ assert _sweepable_cargo_targets(tmp_path / "elsewhere") == ([target.resolve()] if ignored else [])
+
+
def test_collect_rust_target_dirs_includes_the_shared_cargo_target_dir(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
diff --git a/tools/hogli-commands/hogli_commands/tests/test_worktrees.py b/tools/hogli-commands/hogli_commands/tests/test_worktrees.py
index 5341c13c6446..ec328999f568 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_worktrees.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_worktrees.py
@@ -1,9 +1,11 @@
from __future__ import annotations
import os
+import json
import math
import time
import fcntl
+import shutil
import subprocess
from datetime import UTC, datetime
from pathlib import Path
@@ -12,9 +14,17 @@ import pytest
from unittest.mock import Mock, patch
import click
+from click.testing import CliRunner
from hogli import telemetry
from hogli_commands import worktrees
-from hogli_commands.cargo_targets import unused_legacy_cargo_target, worktrees_running_cargo
+from hogli_commands.cargo_targets import (
+ cargo_target_locked,
+ shared_cargo_targets,
+ unused_cargo_target,
+ unused_cargo_units,
+ unused_legacy_cargo_target,
+ worktrees_running_cargo,
+)
from hogli_commands.devbox import config as devbox_config
from hogli_commands.worktrees import (
Worktree,
@@ -22,16 +32,18 @@ from hogli_commands.worktrees import (
_belongs_to_repo,
_boxes_of_removed_worktrees,
_clean_worktree_boxes,
- _collect_deps_items,
+ _collect_artifacts,
_compute_git_state,
_discover_root_worktrees,
- _du_sizes,
_execute,
+ _git_approved_artifacts,
_in_use,
_include_orphans,
_is_under,
+ _leftover_worktree_dirs,
_match_worktree_boxes,
_parse_cutoff,
+ _parse_mode,
_populate_sizes,
_reclaimed_bytes,
_registered_worktrees,
@@ -118,6 +130,37 @@ class TestUnattendedLock:
lock.close()
return True
+ def test_unattended_tiers_skip_hand_made_worktrees_and_still_clean_the_old_cargo_target(
+ self, tmp_path, monkeypatch
+ ) -> None:
+ hand_made = tmp_path / "projects" / "repo-wt-feature"
+ hand_made.mkdir(parents=True)
+ discovered_sources: list[str] = []
+ real_discover = worktrees._discover_candidate_worktrees
+
+ def discover(*args, **kwargs):
+ found = real_discover(*args, **kwargs)
+ discovered_sources.extend(source for source, _ in found)
+ return []
+
+ legacy_cleanups: list[float] = []
+ monkeypatch.setattr(worktrees, "_discover_candidate_worktrees", discover)
+ monkeypatch.setattr(
+ worktrees, "_registered_worktrees", lambda repo_root: {hand_made.resolve(): {"locked": False, "branch": ""}}
+ )
+ monkeypatch.setattr(worktrees, "_claude_scratchpad_root", lambda: tmp_path / "scratchpads")
+ monkeypatch.setattr(worktrees, "_process_cwds", lambda: {Path("/nowhere")})
+ monkeypatch.setattr(
+ worktrees, "_clean_legacy_cargo_target", lambda cutoff, *args: legacy_cleanups.append(cutoff)
+ )
+ monkeypatch.setattr(worktrees, "_leftover_worktree_dirs", lambda *args: [])
+
+ result = CliRunner().invoke(worktrees.worktrees_clean, ["--unattended", "--dry-run"])
+
+ assert result.exit_code == 0, result.output
+ assert "other" not in discovered_sources
+ assert len(legacy_cleanups) == 1
+
def test_second_run_same_day_is_refused(self) -> None:
assert self._take(dry_run=False)
assert not self._take(dry_run=False)
@@ -179,6 +222,20 @@ class TestParseCutoff:
_parse_cutoff(value)
+ ("value", "expected"),
+ [("deps", "deps"), ("build,deps", "deps,build"), (" BUILD , build ", "build"), ("full", "full")],
+)
+def test_parse_mode_normalizes_kinds(value: str, expected: str) -> None:
+ assert _parse_mode(value) == expected
+
+
[email protected]("value", ["full,deps", "deps,cache", "", ","])
+def test_parse_mode_rejects_full_with_kinds_and_unknown_kinds(value: str) -> None:
+ with pytest.raises(click.BadParameter):
+ _parse_mode(value)
+
+
class TestBelongsToRepo:
def test_path_inside_repo_belongs(self, tmp_path) -> None:
repo = tmp_path / "repo"
@@ -246,8 +303,120 @@ class TestWorktreeRoots:
assert [p.resolve() for p in _discover_root_worktrees(roots["phw"], registry)] == [wt.resolve()]
+ @pytest.mark.parametrize(
+ ("selected_sources", "mode", "expected_other"),
+ [(None, "deps,build", True), (None, "full", False), ({"other"}, "full", True), ({"claude"}, "deps", False)],
+ ids=["artifacts_by_default", "full_skips_by_default", "full_when_named", "other_source_selected"],
+ )
+ def test_worktree_registered_outside_every_tool_root_is_other(
+ self, tmp_path, monkeypatch, selected_sources: set[str] | None, mode: str, expected_other: bool
+ ) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path / "home")
+ monkeypatch.setattr(worktrees, "_claude_scratchpad_root", lambda: tmp_path / "scratchpads")
+ hand_made = tmp_path / "projects" / "repo-wt-feature"
+ hand_made.mkdir(parents=True)
+ registry = {hand_made.resolve(): {"locked": False, "branch": "feature"}}
+
+ discovered = worktrees._discover_candidate_worktrees(tmp_path / "repo", registry, selected_sources, mode)
+
+ assert discovered == ([("other", hand_made.resolve())] if expected_other else [])
+
+ @pytest.mark.parametrize(
+ ("layout", "trusted"),
+ [("private_folder", True), ("others_can_write", False), ("symlink", False), ("missing", False)],
+ )
+ def test_scratchpad_root_is_trusted_only_as_a_private_folder(self, tmp_path, layout: str, trusted: bool) -> None:
+ shared_tmp = tmp_path / "tmp"
+ shared_tmp.mkdir()
+ root = shared_tmp / f"claude-{os.getuid()}"
+ if layout == "symlink":
+ (tmp_path / "someone-elses-src").mkdir()
+ root.symlink_to(tmp_path / "someone-elses-src", target_is_directory=True)
+ elif layout != "missing":
+ root.mkdir(mode=0o700)
+ if layout == "others_can_write":
+ root.chmod(0o777)
+
+ assert worktrees._claude_scratchpad_root(shared_tmp) == (root if trusted else None)
+
+
+def _git_worktree(path: Path, files: dict[str, str], tracked: tuple[str, ...] = ()) -> Path:
+ for relative, content in files.items():
+ (path / relative).parent.mkdir(parents=True, exist_ok=True)
+ (path / relative).write_text(content)
+ path.mkdir(parents=True, exist_ok=True)
+ subprocess.run(["git", "init", "-q"], cwd=path, check=True)
+ if tracked:
+ subprocess.run(["git", "add", "--", *tracked], cwd=path, check=True)
+ return path
+
class TestCollectDepsItems:
+ @pytest.mark.parametrize(
+ ("mode", "orphaned", "expected_removed"),
+ [
+ ("build", False, {"web/dist", ".pytest_cache", "web/app.tsbuildinfo", ".flox/cache/go-build"}),
+ ("deps", False, {"web/node_modules", "scratch/node_modules", "tools/.venv", ".flox/cache/venv"}),
+ # Without its index, git cannot say what the branch tracks, so nothing goes.
+ ("build", True, set()),
+ ("deps", True, set()),
+ ],
+ ids=["build", "deps", "orphaned_build", "orphaned_deps"],
+ )
+ def test_removes_only_untracked_artifacts_and_build_output_that_git_ignores(
+ self, tmp_path, mode: str, orphaned: bool, expected_removed: set[str]
+ ) -> None:
+ worktree = _git_worktree(
+ tmp_path / "wt",
+ {
+ ".gitignore": "dist/\nnode_modules/\n.venv/\n*.tsbuildinfo\n.flox/cache/\n",
+ "web/package.json": "{}",
+ "web/dist/app.js": "",
+ "web/app.tsbuildinfo": "",
+ "web/node_modules/react/index.js": "",
+ "web/build/notes.md": "",
+ "action/package.json": "{}",
+ "action/dist/index.js": "",
+ ".pytest_cache/.gitignore": "*\n",
+ ".pytest_cache/v/cache/lastfailed": "{}",
+ "scratch/node_modules/left-pad/index.js": "",
+ "vendored/node_modules/shim/index.js": "",
+ "tools/.venv/pyvenv.cfg": "",
+ "tools/target/release/notes.md": "",
+ "libs/.venv/pyvenv.cfg": "",
+ "libs/.venv/src/editable-pkg/.git/HEAD": "",
+ ".flox/cache/venv/pyvenv.cfg": "",
+ ".flox/cache/go-build/README": "",
+ ".flox/cache/postgres/data/PG_VERSION": "16",
+ "vendor/repo/.git/HEAD": "",
+ "vendor/repo/package.json": "{}",
+ "vendor/repo/dist/lib.js": "",
+ "site/package.json": "{}",
+ "site/dist/.git/HEAD": "",
+ "site/dist/index.html": "",
+ },
+ tracked=(".gitignore", "web/package.json", "action/package.json"),
+ )
+ # Git adds a file under an ignored folder only by force, as a committed GitHub Action bundle or vendored package needs.
+ subprocess.run(
+ ["git", "add", "-f", "action/dist/index.js", "vendored/node_modules/shim/index.js"],
+ cwd=worktree,
+ check=True,
+ )
+ if orphaned:
+ shutil.rmtree(worktree / ".git")
+ (worktree / ".git").write_text(f"gitdir: {tmp_path / 'pruned-admin-dir'}\n")
+
+ (worktree / "notes" / "draft.md").parent.mkdir()
+ (worktree / "notes" / "draft.md").write_text("")
+ (worktree / "linked").mkdir()
+ (worktree / "linked" / "node_modules").symlink_to(worktree / "notes", target_is_directory=True)
+
+ artifacts = _collect_artifacts(worktree, frozenset({mode}))
+ removed = _git_approved_artifacts(worktree, artifacts)
+
+ assert {path.relative_to(worktree.resolve()).as_posix() for path in removed} == expected_removed
+
def test_never_escapes_worktree_via_symlinked_parent(self, tmp_path) -> None:
worktree = tmp_path / "wt"
(worktree / "node_modules").mkdir(parents=True)
@@ -255,9 +424,10 @@ class TestCollectDepsItems:
# `frontend` symlinked to shared external storage that has its own dist/.
external = tmp_path / "external"
(external / "dist").mkdir(parents=True)
+ (external / "package.json").write_text("{}")
(worktree / "frontend").symlink_to(external, target_is_directory=True)
- items = _collect_deps_items(worktree)
+ items = [artifact.path for artifact in _collect_artifacts(worktree, frozenset({"deps", "build"}))]
worktree_resolved = worktree.resolve()
# The real node_modules is collected; nothing resolves outside the worktree.
@@ -268,7 +438,7 @@ class TestCollectDepsItems:
def test_deps_mode_keeps_a_cargo_target_built_after_the_cutoff(self, tmp_path) -> None:
cutoff = time.time() - 7 * 86400
worktree = tmp_path / "wt"
- (worktree / "node_modules").mkdir(parents=True)
+ _git_worktree(worktree, {"node_modules/left-pad/index.js": ""})
stale_target = _write_cargo_target(worktree / "rust" / "target", cutoff - 86400)
# Cargo rewrites fingerprint files in place, so a fresh build leaves the directory mtimes old.
_write_cargo_target(worktree / "cli" / "target", time.time())
@@ -277,7 +447,7 @@ class TestCollectDepsItems:
_populate_sizes([wt], "deps", cutoff)
- assert sorted(wt.deps_items) == sorted([(worktree / "node_modules").resolve(), stale_target.resolve()])
+ assert sorted(wt.artifact_items) == sorted([(worktree / "node_modules").resolve(), stale_target.resolve()])
def _write_cargo_target(target: Path, last_build_epoch: float, profile: Path = Path("debug")) -> Path:
@@ -294,6 +464,21 @@ def _write_cargo_target(target: Path, last_build_epoch: float, profile: Path = P
return target
+def _write_cargo_unit(
+ profile: Path, name: str, unit_hash: str, fingerprint: int, depends_on: list[int], built_epoch: float
+) -> Path:
+ unit = profile / ".fingerprint" / f"{name}-{unit_hash}"
+ unit.mkdir(parents=True)
+ (unit / f"lib-{name}").write_text(fingerprint.to_bytes(8, "little").hex())
+ (unit / f"lib-{name}.json").write_text(json.dumps({"deps": [[0, "dep", False, dep] for dep in depends_on]}))
+ for entry in unit.iterdir():
+ os.utime(entry, (built_epoch, built_epoch))
+ output = profile / "deps" / f"lib{name.replace('-', '_')}-{unit_hash}.rlib"
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_bytes(b"rlib")
+ return output
+
+
def _fake_cargo_processes(monkeypatch, cargo_cwd: Path | None, lsof_available: bool = True) -> None:
def fake_run(args, **kwargs):
if args[0] == "pgrep":
@@ -307,6 +492,35 @@ def _fake_cargo_processes(monkeypatch, cargo_cwd: Path | None, lsof_available: b
monkeypatch.setattr("hogli_commands.cargo_targets.subprocess.run", fake_run)
+class TestLeftoverWorktreeDirs:
+ @pytest.mark.parametrize(
+ ("relative", "extra_entry", "expected_leftover"),
+ [
+ (".claude/worktrees/gone", None, True),
+ (".claude/worktrees/gone", ".git", False),
+ (".claude/worktrees/gone", "docker/compose.yml", False),
+ (".claude/worktrees/gone", ".flox/cache/postgres/PG_VERSION", False),
+ ("home/.codex/worktrees/1a2b/repo", None, True),
+ ("home/.codex/worktrees/1a2b/other-repo", None, False),
+ ],
+ ids=["only_go_cache", "still_a_worktree", "other_files", "flox_state", "home_root", "other_repos_name"],
+ )
+ def test_folder_counts_only_when_it_holds_nothing_but_flox_caches(
+ self, tmp_path, monkeypatch, relative: str, extra_entry: str | None, expected_leftover: bool
+ ) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path / "home")
+ repo = tmp_path / "repo"
+ folder = tmp_path / relative if relative.startswith("home/") else repo / relative
+ (folder / ".flox" / "cache" / "go" / "pkg" / "mod").mkdir(parents=True)
+ if extra_entry is not None:
+ (folder / extra_entry).parent.mkdir(parents=True, exist_ok=True)
+ (folder / extra_entry).write_text("")
+
+ leftovers = _leftover_worktree_dirs(repo, selected_sources=None, cutoff=time.time() + 60)
+
+ assert leftovers == ([folder] if expected_leftover else [])
+
+
class TestCargoTargets:
@pytest.mark.parametrize("lsof_available", [True, False], ids=["traced", "lsof_unavailable"])
def test_worktree_with_cargo_running_below_its_root_is_busy(self, tmp_path, monkeypatch, lsof_available) -> None:
@@ -321,7 +535,7 @@ class TestCargoTargets:
@pytest.mark.parametrize(
"cargo_target_dir_set,days_since_build,cargo_running,expected_offered",
- [(False, 10, False, True), (True, 10, False, False), (False, 1, False, False), (False, 10, True, False)],
+ [(False, 10, False, True), (True, 10, False, True), (False, 1, False, False), (False, 10, True, False)],
ids=["unused", "cargo_target_dir_set", "built_recently", "cargo_running_elsewhere"],
)
def test_legacy_cargo_target_is_offered_only_when_nothing_builds_there(
@@ -342,9 +556,87 @@ class TestCargoTargets:
else:
monkeypatch.delenv("CARGO_TARGET_DIR", raising=False)
- offered = unused_legacy_cargo_target(time.time() - 7 * 86400)
+ cutoff = time.time() - 7 * 86400
+
+ assert unused_legacy_cargo_target(cutoff) == (target if expected_offered and not cargo_target_dir_set else None)
+ assert [shared.path for shared in shared_cargo_targets() if unused_cargo_target(shared.path, cutoff)] == (
+ [target] if expected_offered else []
+ )
+
+ @pytest.mark.parametrize(
+ "case", ["graph", "unreadable_fingerprint", "locked_without_compiling", "symlinked_target"]
+ )
+ def test_sweep_keeps_every_unit_a_recent_build_reaches(self, tmp_path, case: str) -> None:
+ now = time.time()
+ month_ago = now - 30 * 86400
+ target = tmp_path / "target"
+ profile = target / "debug"
+ app_built = month_ago if case == "locked_without_compiling" else now
+ _write_cargo_unit(profile, "app", "a" * 16, fingerprint=1, depends_on=[2], built_epoch=app_built)
+ _write_cargo_unit(profile, "serde", "b" * 16, fingerprint=2, depends_on=[], built_epoch=month_ago)
+ left_pad = _write_cargo_unit(profile, "left-pad", "c" * 16, fingerprint=3, depends_on=[], built_epoch=month_ago)
+ stale_incremental = profile / "incremental" / "left_pad-0k1rn5ce8ga0p"
+ recent_incremental = profile / "incremental" / "app-2k0klfowy1csk"
+ for incremental, epoch in ((stale_incremental, month_ago), (recent_incremental, app_built)):
+ (incremental / "s-session").mkdir(parents=True)
+ os.utime(incremental / "s-session", (epoch, epoch))
+ (profile / ".cargo-lock").write_text("")
+ source_checkout = tmp_path / "src" / "old-module"
+ (source_checkout / "lib.rs").parent.mkdir(parents=True)
+ (source_checkout / "lib.rs").write_text("")
+ os.utime(source_checkout / "lib.rs", (month_ago, month_ago))
+ _write_cargo_unit(target / "release", "app", "d" * 16, fingerprint=4, depends_on=[], built_epoch=now)
+ (target / "release" / "incremental").symlink_to(tmp_path / "src", target_is_directory=True)
+ if case == "unreadable_fingerprint":
+ (profile / ".fingerprint" / f"serde-{'b' * 16}" / "lib-serde.json").write_text("{not json")
+ swept = target
+ if case == "symlinked_target":
+ swept = tmp_path / "linked-target"
+ swept.symlink_to(target, target_is_directory=True)
+
+ unused = unused_cargo_units(swept, now - 7 * 86400)
+
+ resolved_profile = profile.resolve()
+ expected = {
+ "graph": {
+ resolved_profile / ".fingerprint" / f"left-pad-{'c' * 16}",
+ resolved_profile / "deps" / left_pad.name,
+ resolved_profile / "incremental" / stale_incremental.name,
+ },
+ "unreadable_fingerprint": None,
+ "locked_without_compiling": set(),
+ }
+ expected["symlinked_target"] = expected["graph"]
+ assert (set(unused) if unused is not None else None) == expected[case]
+
+ def test_target_lock_waits_for_a_build_and_blocks_one_while_held(self, tmp_path) -> None:
+ target = _write_cargo_target(tmp_path / "target", time.time())
+ cargo_lock = target / "debug" / ".cargo-lock"
+ cargo_lock.write_text("")
+
+ with cargo_lock.open("a") as build:
+ fcntl.flock(build, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ with cargo_target_locked(target) as locked:
+ assert not locked
+
+ with cargo_target_locked(target) as locked, cargo_lock.open("a") as build:
+ assert locked
+ with pytest.raises(BlockingIOError):
+ fcntl.flock(build, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+ @pytest.mark.parametrize("broken_git_metadata", [False, True], ids=["checkout", "checkout_git_cannot_read"])
+ def test_cargo_target_dir_inside_a_checkout_is_not_treated_as_shared(
+ self, tmp_path, monkeypatch, broken_git_metadata: bool
+ ) -> None:
+ checkout = _git_worktree(tmp_path / "checkout", {})
+ if broken_git_metadata:
+ shutil.rmtree(checkout / ".git")
+ (checkout / ".git").write_text(f"gitdir: {tmp_path / 'pruned-admin-dir'}\n")
+ target = _write_cargo_target(checkout / "target", time.time() - 10 * 86400)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path / "home")
+ monkeypatch.setenv("CARGO_TARGET_DIR", str(target))
- assert offered == (target if expected_offered else None)
+ assert [shared.path for shared in shared_cargo_targets()] == []
class TestRegisteredWorktrees:
@@ -873,6 +1165,29 @@ class TestExecute:
assert second.exists()
assert [call.args[0].path for call in before_remove.call_args_list] == [first]
+ @pytest.mark.parametrize(
+ ("case", "expected_removed_failed"),
+ [("removable", (1, 0)), ("repo_deep_inside", (0, 1)), ("tracked_during_prompt", (0, 0))],
+ )
+ def test_artifact_is_checked_again_just_before_removal(
+ self, tmp_path, case: str, expected_removed_failed: tuple[int, int]
+ ) -> None:
+ worktree = _git_worktree(
+ tmp_path / "wt", {".gitignore": "node_modules/\n", "node_modules/pkg/index.js": ""}, tracked=(".gitignore",)
+ )
+ node_modules = (worktree / "node_modules").resolve()
+ if case == "repo_deep_inside":
+ (node_modules / ".pnpm" / "[email protected]" / "node_modules" / "pkg" / "src" / ".git").mkdir(parents=True)
+ wt = Worktree(source="claude", path=worktree, registered=True, locked=False, branch="", last_activity=0.0)
+ wt.artifact_items = [node_modules]
+ if case == "tracked_during_prompt":
+ subprocess.run(["git", "add", "-f", "node_modules/pkg/index.js"], cwd=worktree, check=True)
+
+ reap = _execute([wt], "deps", tmp_path)
+
+ assert (reap.removed, reap.failed) == expected_removed_failed
+ assert node_modules.exists() == (case != "removable")
+
@pytest.mark.parametrize("read_only_go_module_cache", [True, False], ids=["read_only_tree", "already_removed"])
def test_full_mode_counts_worktree_as_removed(self, tmp_path, read_only_go_module_cache) -> None:
worktree = tmp_path / "wt"
@@ -891,23 +1206,3 @@ class TestExecute:
assert (reap.removed, reap.failed) == (1, 0)
assert not worktree.exists()
-
-
-class TestDuSizes:
- def test_parses_per_path_and_skips_garbage(self, tmp_path, monkeypatch) -> None:
- a = tmp_path / "a"
- b = tmp_path / "b"
- a.mkdir()
- b.mkdir()
-
- stdout = f"4\t{a}\n\nnot-a-number\t/whatever\n8\t{b}\n"
- monkeypatch.setattr(
- "hogli_commands.worktrees.subprocess.run",
- lambda *a, **k: subprocess.CompletedProcess(args=[], returncode=1, stdout=stdout, stderr=""),
- )
-
- sizes = _du_sizes([a, b])
- assert sizes == {str(a): 4 * 1024.0, str(b): 8 * 1024.0}
-
- def test_no_existing_paths_returns_empty(self, tmp_path) -> None:
- assert _du_sizes([tmp_path / "missing"]) == {}
diff --git a/tools/hogli-commands/hogli_commands/worktrees.py b/tools/hogli-commands/hogli_commands/worktrees.py
index 238faec20ea0..67e4d85ed4e5 100644
--- a/tools/hogli-commands/hogli_commands/worktrees.py
+++ b/tools/hogli-commands/hogli_commands/worktrees.py
@@ -4,7 +4,7 @@ These tools each create throwaway git worktrees that accumulate over time. A
single PostHog worktree carries multi-GB `node_modules`, Python venvs, and Rust
build artifacts, so stale ones are by far the largest reclaimable disk on a dev
machine. This command finds them, filters by age, and either removes the whole
-worktree or just its dependencies/build artifacts (keeping the code).
+worktree or just its dependencies and build output (keeping the code).
The discovery roots and dependency patterns are PostHog-specific, but the age
logic and deletion modes are generic.
@@ -16,10 +16,12 @@ state it cannot read (present-but-corrupt/locked) unless `--include-dirty` is
given. Orphaned worktrees (admin gitdir pruned) are removed by default: deleting
the directory never touches git refs, so committed work survives in the shared
repo — only uncommitted edits in an already-stale tree could be lost. `--mode
-deps` only removes recreatable artifacts, so it has no such gate — it can strip
-deps from a worktree that is idle by the activity signal but still has a live
-dev server running. Both modes skip a worktree where cargo or rustc runs, and
-`--mode deps` keeps a Cargo target that a build used after the cutoff.
+deps` and `--mode build` only remove recreatable artifacts, so they have no such
+gate, so they can strip a worktree that is idle by the activity signal but still
+has a live dev server running. They never remove a path that holds a tracked
+file, and `build` removes only paths that git ignores. Every mode skips a
+worktree where cargo or rustc runs, and deps and build keep a Cargo target that
+a build used after the cutoff.
"""
from __future__ import annotations
@@ -27,9 +29,11 @@ from __future__ import annotations
import os
import re
import sys
+import stat
import time
import fcntl
import shutil
+import itertools
import subprocess
from collections.abc import Callable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
@@ -42,7 +46,12 @@ import click
from hogli import telemetry
from hogli.manifest import REPO_ROOT
-from .cargo_targets import last_cargo_build_epoch, unused_legacy_cargo_target, worktrees_running_cargo
+from .cargo_targets import (
+ last_cargo_build_epoch,
+ looks_like_cargo_target,
+ unused_legacy_cargo_target,
+ worktrees_running_cargo,
+)
from .devbox import hogland, mutagen
from .devbox.coder import (
delete_workspace,
@@ -60,24 +69,53 @@ from .devbox.config import (
read_worktree_id,
saved_worktree_box_cleanup,
)
+from .disk_usage import NO_USAGE, DiskUsage, measure_disk_usage
from .doctor import _format_size
+WORKTREE_SOURCES = ("claude", "claude-scratchpad", "codex", "posthog-code", "phw", "other")
+
# Roots that hold agent-created worktrees, keyed by the tool that owns them.
-# Claude Code and Codex nest worktrees directly under the repo; PostHog Desktop
-# keeps them in the home directory as <id>/posthog. Returned as (source, root).
-# The phw root must match WORKTREE_BASE in bin/posthog-worktree, including its
+# Claude Code nests worktrees under the repo. The Codex app and PostHog Desktop
+# keep them in the home directory as <id>/<repo>, and Claude Code agents make
+# throwaway ones in their scratchpad. Returned as (source, root). The phw root
+# must match WORKTREE_BASE in bin/posthog-worktree, including its
# POSTHOG_WORKTREE_BASE override.
def _worktree_roots(repo_root: Path) -> list[tuple[str, Path]]:
phw_base = os.environ.get("POSTHOG_WORKTREE_BASE") or str(Path.home() / ".worktrees" / "posthog")
+ scratchpad_root = _claude_scratchpad_root()
return [
("claude", repo_root / ".claude" / "worktrees"),
+ *([("claude-scratchpad", scratchpad_root)] if scratchpad_root is not None else []),
("codex", repo_root / ".codex" / "worktrees"),
+ ("codex", repo_root / ".codex-worktrees"),
+ ("codex", Path.home() / ".codex" / "worktrees"),
("posthog-code", Path.home() / ".posthog-code" / "worktrees"),
("phw", Path(phw_base).expanduser()),
]
+def _claude_scratchpad_root(shared_tmp: Path = Path("/tmp")) -> Path | None:
+ """The scratchpad root, or None unless it is a real folder that only this user owns and can write to.
+
+ /tmp is shared, so another user can create this name first, or make it a
+ symlink that points discovery at someone else's folders.
+ """
+
+ root = shared_tmp / f"claude-{os.getuid()}"
+ try:
+ status = root.lstat()
+ except OSError:
+ return None
+ if not stat.S_ISDIR(status.st_mode) or status.st_uid != os.getuid() or status.st_mode & 0o022:
+ return None
+ return root
+
+
+# Claude Code's scratchpad worktrees sit at <root>/<encoded project>/<session id>/scratchpad/<worktree>.
+_SCRATCHPAD_DISCOVERY_DEPTH = 4
+
+
def worktree_source(path: Path) -> str:
"""Name the tool whose worktree root holds *path*, or ``other`` for a worktree made elsewhere."""
return _source_under(_resolve(path), _resolve_repo(str(path)))
@@ -90,55 +128,70 @@ def _source_under(resolved_path: Path, repo_root: Path) -> str:
return "other"
-# Dependency/build-artifact directory names removed in --mode deps. Matched at
-# any depth and never descended into once matched, so walking stays cheap.
-DEPS_RECURSIVE_DIRS = frozenset(
- {
- "node_modules",
- "target", # Cargo — guarded by _looks_like_cargo_target
- "__pycache__",
- ".pytest_cache",
- ".mypy_cache",
- ".ruff_cache",
- ".parcel-cache",
- ".turbo",
- }
-)
+ARTIFACT_KINDS = ("deps", "build")
+_DEPS = frozenset({"deps"})
+_BUILD = frozenset({"build"})
-# Directories never walked into when collecting deps — either handled explicitly
-# below or off-limits (git metadata).
-DEPS_WALK_SKIP = frozenset({".git", ".flox", ".venv", "venv"})
-
-# Explicit relative paths (globs allowed) removed in --mode deps. These are not
-# matched recursively, so they must name their location from the worktree root.
-DEPS_EXPLICIT_PATHS = (
- ".venv",
- "venv",
- ".flox/cache/venv",
- ".flox/cache/go",
- ".flox/cache/go-build",
- "frontend/dist",
- "frontend/.cache",
- "frontend/tmp",
- "frontend/storybook-static",
- "storybook-static",
- "playwright-report",
- "playwright/test-results",
- "test-results",
- "products/*/dist",
- "products/*/storybook-static",
-)
-# Leaf names of the explicit artifact paths above — pruned from os.walk so the
-# walk doesn't descend their interiors before the explicit-glob pass deletes
-# them wholesale.
-DEPS_WALK_PRUNE_LEAVES = frozenset({"dist", ".cache", "tmp", "storybook-static", "playwright-report", "test-results"})
+def _is_python_venv(path: Path) -> bool:
+ return (path / "pyvenv.cfg").is_file()
-_INTERVAL_UNITS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
-# du can hang on a stale network mount; bound it so one bad path can't stall the
-# whole sizing pool.
-_DU_TIMEOUT_SECONDS = 120.0
+_BUILD_MANIFESTS = ("package.json", "pyproject.toml", "setup.py", "setup.cfg", "Cargo.toml", "go.mod", "CMakeLists.txt")
+
+
+def _beside_build_manifest(path: Path) -> bool:
+ return any((path.parent / manifest).is_file() for manifest in _BUILD_MANIFESTS)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class _ArtifactRule:
+ kinds: frozenset[str]
+ applies_to: Callable[[Path], bool] = lambda path: True
+
+
+ARTIFACT_DIRS: dict[str, _ArtifactRule] = {
+ "node_modules": _ArtifactRule(kinds=_DEPS),
+ ".venv": _ArtifactRule(kinds=_DEPS, applies_to=_is_python_venv),
+ "venv": _ArtifactRule(kinds=_DEPS, applies_to=_is_python_venv),
+ ".tox": _ArtifactRule(kinds=_DEPS),
+ ".nox": _ArtifactRule(kinds=_DEPS),
+ ".pnpm-store": _ArtifactRule(kinds=_DEPS),
+ # Cargo builds the dependencies and the repo's own crates into the same target dir.
+ "target": _ArtifactRule(kinds=_DEPS | _BUILD, applies_to=looks_like_cargo_target),
+ "__pycache__": _ArtifactRule(kinds=_BUILD),
+ ".pytest_cache": _ArtifactRule(kinds=_BUILD),
+ ".mypy_cache": _ArtifactRule(kinds=_BUILD),
+ ".ruff_cache": _ArtifactRule(kinds=_BUILD),
+ ".parcel-cache": _ArtifactRule(kinds=_BUILD),
+ ".turbo": _ArtifactRule(kinds=_BUILD),
+ ".next": _ArtifactRule(kinds=_BUILD),
+ ".nuxt": _ArtifactRule(kinds=_BUILD),
+ ".svelte-kit": _ArtifactRule(kinds=_BUILD),
+ ".vite": _ArtifactRule(kinds=_BUILD),
+ ".react-router": _ArtifactRule(kinds=_BUILD),
+ "storybook-static": _ArtifactRule(kinds=_BUILD),
+ "playwright-report": _ArtifactRule(kinds=_BUILD),
+ "test-results": _ArtifactRule(kinds=_BUILD),
+ "blob-report": _ArtifactRule(kinds=_BUILD),
+ "htmlcov": _ArtifactRule(kinds=_BUILD),
+ # These names are generic, so they count only as the output of a package built in the same directory.
+ "coverage": _ArtifactRule(kinds=_BUILD, applies_to=_beside_build_manifest),
+ "dist": _ArtifactRule(kinds=_BUILD, applies_to=_beside_build_manifest),
+ "build": _ArtifactRule(kinds=_BUILD, applies_to=_beside_build_manifest),
+}
+
+# Flox keeps environment state such as a local Postgres data dir under .flox/cache, so only these children are artifacts.
+FLOX_CACHE_ARTIFACTS: dict[str, frozenset[str]] = {"venv": _DEPS, "go": _DEPS, "go-build": _BUILD}
+
+# A .tsbuildinfo left behind makes `tsc --build` skip emitting into an outDir that was removed.
+BUILD_FILE_SUFFIXES = (".tsbuildinfo", ".node")
+
+BUILD_EXPLICIT_PATHS = ("frontend/.cache", "frontend/tmp")
+
+_WALK_SKIP = frozenset({".git", ".venv", "venv"})
+
+_INTERVAL_UNITS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
@dataclass(frozen=True, kw_only=True, slots=True)
@@ -158,9 +211,8 @@ class Worktree:
locked: bool
branch: str
last_activity: float
- size: float = 0.0
- deps_items: list[Path] = field(default_factory=list)
- deps_sizes: dict[str, float] = field(default_factory=dict)
+ usage: DiskUsage = NO_USAGE
+ artifact_items: list[Path] = field(default_factory=list)
dirty: bool = False
unpushed: int = 0
detached: bool = False
@@ -212,16 +264,19 @@ class WorktreeBox:
)
@click.option(
"--mode",
- type=click.Choice(["full", "deps"], case_sensitive=False),
default=None,
- help="Required. 'full' removes the whole worktree; 'deps' removes only deps/build artifacts, keeping the code.",
+ metavar="full|deps|build",
+ help="Required. 'full' removes the whole worktree. 'deps' removes installed dependencies such as node_modules, "
+ "Python venvs and Cargo target dirs. 'build' removes build output and tool caches that git ignores, such as "
+ "Cargo target, dist, .next and __pycache__. Combine deps and build with a comma: --mode deps,build.",
)
@click.option(
"--source",
"sources",
multiple=True,
- type=click.Choice(["claude", "codex", "posthog-code", "phw"], case_sensitive=False),
- help="Restrict to specific tool(s). Repeatable. Default: all of them.",
+ type=click.Choice(WORKTREE_SOURCES, case_sensitive=False),
+ help="Restrict to specific tool(s). Repeatable. 'other' is a worktree that `git worktree list` registers "
+ "outside every tool's folder. Default: all of them, except that --mode full skips 'other' unless you name it.",
)
@click.option(
"--repo",
@@ -242,7 +297,8 @@ class WorktreeBox:
default=None,
help="What to do with orphaned worktrees (admin entry pruned, not in `git worktree list`; "
"committed work is safe in branch refs). yes=include, no=skip, ask=prompt when any are found. "
- "Default: yes for --mode deps, ask for --mode full.",
+ "Default: yes for --mode deps and build, ask for --mode full. deps and build still remove nothing from an "
+ "orphan, because git cannot check what its branch tracks.",
)
@click.option(
"--stop-boxes",
@@ -265,9 +321,10 @@ class WorktreeBox:
@click.option(
"--unattended",
is_flag=True,
- help="For a scheduler: once a day, remove worktrees idle for 14 days, then strip deps from worktrees "
+ help="For a scheduler: once a day, remove worktrees idle for 14 days, then strip deps and build output from worktrees "
"idle for 7 days. Implies --yes and --keep-boxes.",
)
[email protected]("--skip-hand-made", is_flag=True, hidden=True)
@click.pass_context
def worktrees_clean(
ctx: click.Context,
@@ -283,6 +340,7 @@ def worktrees_clean(
dry_run: bool,
yes: bool,
unattended: bool,
+ skip_hand_made: bool = False,
) -> None:
"""Find and clean up stale agent worktrees by age and deletion mode."""
@@ -305,6 +363,8 @@ def worktrees_clean(
mode=tier,
repo=repo,
sources=sources,
+ # A worktree outside every tool folder is often a long-lived checkout that a person made by hand.
+ skip_hand_made=True,
orphans=orphans,
dry_run=dry_run,
yes=True,
@@ -320,12 +380,13 @@ def worktrees_clean(
"Both --before and --mode are required.\n\n"
"Examples:\n"
" hogli worktrees:clean --before 7d --mode deps # free deps in worktrees idle >7 days\n"
+ " hogli worktrees:clean --before 7d --mode deps,build\n"
" hogli worktrees:clean --before 2026-06-01 --mode full\n"
" hogli worktrees:clean --before all --mode full --dry-run\n"
" hogli worktrees:clean --before 30d --mode full --repo ~/Projects/other-repo"
)
- mode = mode.lower()
+ mode = _parse_mode(mode)
box_action = _resolve_box_action(stop_boxes, delete_boxes, keep_boxes, mode)
orphan_policy = _resolve_orphan_policy(orphans, mode)
repo_root = _resolve_repo(repo)
@@ -337,7 +398,7 @@ def worktrees_clean(
pending = _boxes_of_removed_worktrees(load_worktree_boxes(), repo_root, repo_common, selected_sources)
_clean_boxes_of_removed_worktrees(pending, box_action, cutoff, dry_run, yes)
# ~/.cargo/target belongs to no single worktree or repository, so a run narrowed by --source or --repo leaves it alone.
- if mode == "deps" and selected_sources is None and repo is None:
+ if mode != "full" and selected_sources is None and repo is None:
_clean_legacy_cargo_target(cutoff, dry_run, yes)
protected = _protected_paths(repo_root)
@@ -352,35 +413,36 @@ def worktrees_clean(
if busy is None:
click.echo("Could not list running processes, so no worktree can be shown to be idle; aborting to stay safe.")
return
+ leftovers = [
+ path for path in _leftover_worktree_dirs(repo_root, selected_sources, cutoff) if not _in_use(path, busy)
+ ]
+ _clean_leftover_worktree_dirs(leftovers, dry_run, yes)
in_use: list[Path] = []
candidates: list[Worktree] = []
- for source, root in _worktree_roots(repo_root):
- if selected_sources is not None and source not in selected_sources:
+ for source, path in _discover_candidate_worktrees(repo_root, registry, selected_sources, mode, skip_hand_made):
+ resolved = _resolve(path)
+ if resolved in protected:
continue
- for path in _discover_root_worktrees(root, registry):
- resolved = _resolve(path)
- if resolved in protected:
- continue
- # PostHog Desktop worktrees live outside the repo and the home root is
- # shared across repos, so only act on worktrees owned by this repo.
- if not _belongs_to_repo(resolved, repo_root, repo_common):
- continue
- meta = registry.get(resolved)
- if meta and meta["locked"]:
- continue # never touch a git-locked worktree
- if _in_use(resolved, busy):
- in_use.append(path)
- continue
- candidates.append(
- Worktree(
- source=source,
- path=path,
- registered=meta is not None,
- locked=bool(meta and meta["locked"]),
- branch=(meta["branch"] if meta else ""),
- last_activity=_last_activity(path),
- )
+ # PostHog Desktop worktrees live outside the repo and the home root is
+ # shared across repos, so only act on worktrees owned by this repo.
+ if not _belongs_to_repo(resolved, repo_root, repo_common):
+ continue
+ meta = registry.get(resolved)
+ if meta and meta["locked"]:
+ continue # never touch a git-locked worktree
+ if _in_use(resolved, busy):
+ in_use.append(path)
+ continue
+ candidates.append(
+ Worktree(
+ source=source,
+ path=path,
+ registered=meta is not None,
+ locked=bool(meta and meta["locked"]),
+ branch=(meta["branch"] if meta else ""),
+ last_activity=_last_activity(path),
)
+ )
if in_use:
click.echo(f"Skipping {len(in_use)} worktree(s) that a running process has as its working directory:")
@@ -434,15 +496,18 @@ def worktrees_clean(
for wt in sorted(stale, key=lambda w: w.last_activity):
markers = _listing_markers(wt, mode)
click.echo(
- f" {_format_size(wt.size):>10} {_ago(wt.last_activity):>12} ago {markers} {_display_path(wt.path)}"
+ f" {_format_size(wt.usage.freeable_bytes):>10} {_ago(wt.last_activity):>12} ago {markers} {_display_path(wt.path)}"
)
- total = sum(wt.size for wt in stale)
- action = "fully remove" if mode == "full" else "strip deps/build artifacts from"
- click.echo(
- f"\nWould {action} {len(stale)} worktree(s); estimated up to ~{_format_size(total)} "
- "(cloned/hardlinked deps free less — the actual reclaimed total is measured after deletion)."
- )
+ total = sum((wt.usage for wt in stale), NO_USAGE)
+ action = "fully remove" if mode == "full" else f"strip {_describe_artifact_kinds(mode)} from"
+ click.echo(f"\nWould {action} {len(stale)} worktree(s), which frees about {_format_size(total.freeable_bytes)}.")
+ shared_bytes = total.allocated_bytes - total.freeable_bytes
+ if shared_bytes > 0:
+ click.echo(
+ f"Another {_format_size(shared_bytes)} in them shares its blocks with copies elsewhere, "
+ "such as the pnpm or uv store, so deleting it frees nothing."
+ )
registered_boxes = _match_worktree_boxes(stale, load_worktree_boxes()) if mode == "full" else []
boxes = registered_boxes if box_action else []
@@ -465,7 +530,7 @@ def worktrees_clean(
# A build can start, or finish, while the prompt waits.
stale = _without_cargo_builds(stale)
for wt in stale:
- wt.deps_items = [item for item in wt.deps_items if not _cargo_built_since(item, cutoff)]
+ wt.artifact_items = [item for item in wt.artifact_items if not _cargo_built_since(item, cutoff)]
if not stale:
click.echo("Nothing left to remove.")
return
@@ -503,7 +568,7 @@ def _resolve_box_action(stop_boxes: bool, delete_boxes: bool, keep_boxes: bool,
raise click.UsageError("Pass at most one of --stop-boxes, --delete-boxes, --keep-boxes.")
if (stop_boxes or delete_boxes) and mode != "full":
raise click.UsageError(
- "--stop-boxes and --delete-boxes need --mode full, because --mode deps keeps the worktree."
+ f"--stop-boxes and --delete-boxes need --mode full, because --mode {mode} keeps the worktree."
)
if keep_boxes or mode != "full":
return None
@@ -810,15 +875,15 @@ def _clean_coder_boxes(
def _resolve_orphan_policy(explicit: str | None, mode: str) -> str:
- """Resolve --orphans, defaulting per mode: deps cleans orphans (their deps are
- just recreatable artifacts), full prompts (it deletes the working copy)."""
+ """Resolve --orphans, defaulting per mode: deps and build clean orphans (they
+ remove only recreatable artifacts), full prompts (it deletes the working copy)."""
if explicit is not None:
return explicit.lower()
return "ask" if mode == "full" else "yes"
-_UNATTENDED_SCHEDULE = (("14d", "full"), ("7d", "deps"))
+_UNATTENDED_SCHEDULE = (("14d", "full"), ("7d", "deps,build"))
# .flox/env/on-activate.sh checks this file's mtime before it spawns hogli.
_UNATTENDED_LOCK = Path.home() / ".config" / "posthog" / "worktrees-clean.lock"
@@ -932,6 +997,29 @@ def _git_common_dir(start: Path) -> Path | None:
return _resolve(common)
+def _parse_mode(value: str) -> str:
+ """Normalize --mode to ``full`` or a comma list of artifact kinds in a fixed order, such as ``deps,build``."""
+
+ parts = {part.strip().lower() for part in value.split(",") if part.strip()}
+ unknown = parts - {"full", *ARTIFACT_KINDS}
+ if not parts or unknown:
+ raise click.BadParameter(f"{value!r} is not full, deps, build, or a comma list of deps and build.")
+ if "full" in parts:
+ if len(parts) > 1:
+ raise click.BadParameter("full removes the whole worktree, so it cannot be combined with deps or build.")
+ return "full"
+ return ",".join(kind for kind in ARTIFACT_KINDS if kind in parts)
+
+
+def _artifact_kinds(mode: str) -> frozenset[str]:
+ return frozenset(mode.split(",")) & frozenset(ARTIFACT_KINDS)
+
+
+def _describe_artifact_kinds(mode: str) -> str:
+ labels = {"deps": "dependencies", "build": "build output"}
+ return " and ".join(labels[kind] for kind in ARTIFACT_KINDS if kind in _artifact_kinds(mode))
+
+
def _parse_cutoff(value: str) -> tuple[float, str]:
"""Parse --before into (cutoff_epoch, human_label). Raises click.BadParameter on garbage."""
@@ -988,8 +1076,36 @@ def _discover_worktrees(root: Path, max_depth: int = 2) -> list[Path]:
return found
+def _discover_candidate_worktrees(
+ repo_root: Path,
+ registry: dict[Path, dict],
+ selected_sources: set[str] | None,
+ mode: str,
+ skip_hand_made: bool = False,
+) -> list[tuple[str, Path]]:
+ """Worktrees under the known tool roots, then the ones that `git worktree list` registers anywhere else.
+
+ A worktree outside every tool root is one a person made by hand, so --mode
+ full and the unattended run act on it only when --source other asks for it.
+ """
+
+ discovered: list[tuple[str, Path]] = []
+ resolved_roots = [_resolve(root) for _, root in _worktree_roots(repo_root)]
+ for source, root in _worktree_roots(repo_root):
+ if selected_sources is None or source in selected_sources:
+ discovered.extend((source, path) for path in _discover_root_worktrees(root, registry))
+ include_other = (
+ "other" in selected_sources if selected_sources is not None else not (mode == "full" or skip_hand_made)
+ )
+ if include_other:
+ for path in registry:
+ if path.is_dir() and not any(_is_under(path, root) for root in resolved_roots):
+ discovered.append(("other", path))
+ return discovered
+
+
def _discover_root_worktrees(root: Path, registry: dict[Path, dict]) -> list[Path]:
- found = _discover_worktrees(root)
+ found = _discover_worktrees(root, max_depth=_SCRATCHPAD_DISCOVERY_DEPTH if root == _claude_scratchpad_root() else 2)
seen = {_resolve(path) for path in found}
resolved_root = _resolve(root)
for path in registry:
@@ -1033,6 +1149,79 @@ def _registered_worktrees(repo_root: Path) -> dict[Path, dict] | None:
return registry
+def _leftover_worktree_dirs(repo_root: Path, selected_sources: set[str] | None, cutoff: float) -> list[Path]:
+ """Folders where a worktree was, that now hold only the flox caches its removal could not delete.
+
+ Go writes its module cache without write permission, which stopped earlier
+ removals part way. A worktree root in the home directory is shared across
+ repos, so there a folder counts only when it carries this repo's name, as
+ the worktrees there do.
+ """
+
+ leftovers: list[Path] = []
+ for source, root in _worktree_roots(repo_root):
+ if source in ("phw", "claude-scratchpad") or (selected_sources is not None and source not in selected_sources):
+ continue
+ if _is_under(_resolve(root), repo_root):
+ candidates = _child_dirs(root)
+ else:
+ candidates = [child / repo_root.name for child in _child_dirs(root)]
+ for candidate in candidates:
+ if _holds_only_flox_caches(candidate) and _newest_mtime([candidate, candidate / ".flox"]) < cutoff:
+ leftovers.append(candidate)
+ return leftovers
+
+
+def _child_dirs(directory: Path) -> list[Path]:
+ try:
+ return [child for child in directory.iterdir() if child.is_dir() and not child.is_symlink()]
+ except OSError:
+ return []
+
+
+def _holds_only_flox_caches(directory: Path) -> bool:
+ try:
+ if {entry.name for entry in directory.iterdir()} != {".flox"}:
+ return False
+ if {entry.name for entry in (directory / ".flox").iterdir()} != {"cache"}:
+ return False
+ cache_entries = {entry.name for entry in (directory / ".flox" / "cache").iterdir()}
+ except OSError:
+ return False
+ return bool(cache_entries) and cache_entries <= FLOX_CACHE_ARTIFACTS.keys()
+
+
+def _newest_mtime(paths: Sequence[Path]) -> float:
+ try:
+ return max(path.stat().st_mtime for path in paths)
+ except OSError:
+ return float("inf")
+
+
+def _clean_leftover_worktree_dirs(leftovers: list[Path], dry_run: bool, yes: bool) -> None:
+ if not leftovers:
+ return
+ size = measure_disk_usage(leftovers).freeable_bytes
+ click.echo(
+ f"{len(leftovers)} folder(s) of removed worktrees hold only flox caches ({_format_size(size)} to free), "
+ "because Go's read-only module cache stopped their removal:"
+ )
+ for leftover in leftovers[:5]:
+ click.echo(f" {_display_path(leftover)}")
+ if len(leftovers) > 5:
+ click.echo(f" … and {len(leftovers) - 5} more")
+ if dry_run:
+ click.echo("[DRY-RUN] Not deleted.\n")
+ return
+ if not yes and not click.confirm("Delete them?", default=False):
+ click.echo()
+ return
+ failed = _delete_paths(leftovers)
+ for leftover in leftovers:
+ _cleanup_empty_parent(leftover)
+ click.echo(f"✓ Deleted {len(leftovers) - failed} folder(s).\n")
+
+
def _protected_paths(repo_root: Path) -> set[Path]:
"""Worktrees that must never be deleted: the main repo and the current one."""
@@ -1272,8 +1461,16 @@ def _repo_has_remotes(repo_root: Path) -> bool:
def _git(cwd: Path, args: Sequence[str], timeout: float = 30.0) -> subprocess.CompletedProcess[str] | None:
+ # A plain `git status` refreshes the index, and the index mtime is one of the worktree's activity signals.
try:
- return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, timeout=timeout)
+ return subprocess.run(
+ ["git", "--no-optional-locks", *args],
+ cwd=cwd,
+ capture_output=True,
+ text=True,
+ errors="surrogateescape",
+ timeout=timeout,
+ )
except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
return None
@@ -1310,15 +1507,15 @@ def _listing_markers(wt: Worktree, mode: str) -> str:
def _populate_sizes(worktrees: list[Worktree], mode: str, cutoff: float) -> None:
- """Fill in .size (and deps items/sizes for deps mode) using a thread pool."""
+ """Fill in .usage, and the artifact items outside full mode, using a thread pool."""
def work(wt: Worktree) -> None:
if mode == "full":
- wt.deps_sizes = _du_sizes([wt.path])
+ wt.usage = measure_disk_usage([wt.path])
else:
- wt.deps_items = [item for item in _collect_deps_items(wt.path) if not _cargo_built_since(item, cutoff)]
- wt.deps_sizes = _du_sizes(wt.deps_items)
- wt.size = sum(wt.deps_sizes.values())
+ artifacts = _git_approved_artifacts(wt.path, _collect_artifacts(wt.path, _artifact_kinds(mode)))
+ wt.artifact_items = [item for item in artifacts if not _cargo_built_since(item, cutoff)]
+ wt.usage = measure_disk_usage(wt.artifact_items)
if not worktrees:
return
@@ -1326,51 +1523,135 @@ def _populate_sizes(worktrees: list[Worktree], mode: str, cutoff: float) -> None
list(pool.map(work, worktrees))
-def _collect_deps_items(worktree: Path) -> list[Path]:
- """Find dependency/build-artifact directories inside a worktree.
+@dataclass(frozen=True, kw_only=True, slots=True)
+class _Artifact:
+ path: Path
+ kinds: frozenset[str]
+
+
+def _collect_artifacts(worktree: Path, kinds: frozenset[str]) -> list[_Artifact]:
+ """Find the dependency and build-output paths of the selected *kinds* inside a worktree.
Every returned path is resolved and confirmed to stay within the worktree, so
a symlinked component (e.g. a `frontend` symlinked to shared storage) can
- never cause deletion outside the worktree.
+ never cause deletion outside the worktree. The walk does not enter another
+ repository, submodule or worktree nested inside this one.
"""
worktree_resolved = _resolve(worktree)
- items: list[Path] = []
- seen: set[Path] = set()
+ found: dict[Path, frozenset[str]] = {}
- def add(candidate: Path) -> None:
+ def add(candidate: Path, rule_kinds: frozenset[str]) -> None:
+ selected = rule_kinds & kinds
+ if not selected:
+ return
+ # A symlinked artifact name points at another folder, which may be someone's work, so only real folders count.
+ if candidate.is_symlink():
+ return
resolved = _resolve(candidate)
- if not _is_under(resolved, worktree_resolved):
- return # symlink/`..` escape — refuse to delete outside the worktree
- if resolved not in seen and resolved.exists():
- seen.add(resolved)
- items.append(resolved)
-
- for dirpath, dirnames, _ in os.walk(worktree):
- matched: list[str] = []
- for name in list(dirnames):
- if name in DEPS_RECURSIVE_DIRS:
- full = Path(dirpath) / name
- if name == "target" and not _looks_like_cargo_target(full):
- continue
- add(full)
- matched.append(name)
- # Don't descend into matched artifacts, skip-listed dirs, or the
- # explicit-path artifact dirs (handled by the glob pass below).
- dirnames[:] = [
- d for d in dirnames if d not in matched and d not in DEPS_WALK_SKIP and d not in DEPS_WALK_PRUNE_LEAVES
- ]
-
- for pattern in DEPS_EXPLICIT_PATHS:
+ if _is_under(resolved, worktree_resolved) and resolved.exists():
+ found.setdefault(resolved, selected)
+
+ for dirpath, dirnames, filenames in os.walk(worktree):
+ parent = Path(dirpath)
+ descend: list[str] = []
+ for name in dirnames:
+ child = parent / name
+ if (child / ".git").exists():
+ continue
+ if name == ".flox":
+ for cache_child, rule_kinds in FLOX_CACHE_ARTIFACTS.items():
+ add(child / "cache" / cache_child, rule_kinds)
+ continue
+ rule = ARTIFACT_DIRS.get(name)
+ if rule is not None and rule.applies_to(child):
+ add(child, rule.kinds)
+ continue
+ if name in _WALK_SKIP:
+ continue
+ descend.append(name)
+ dirnames[:] = descend
+ for name in filenames:
+ if name.endswith(BUILD_FILE_SUFFIXES):
+ add(parent / name, _BUILD)
+
+ for pattern in BUILD_EXPLICIT_PATHS:
for path in worktree.glob(pattern):
if path.is_dir():
- add(path)
+ add(path, _BUILD)
+
+ outermost = sorted(found, key=lambda path: len(path.parts))
+ kept: list[_Artifact] = []
+ kept_paths: set[Path] = set()
+ for path in outermost:
+ if not any(ancestor in kept_paths for ancestor in path.parents):
+ kept_paths.add(path)
+ kept.append(_Artifact(path=path, kinds=found[path]))
+ return kept
+
+
+def _git_approved_artifacts(worktree: Path, artifacts: list[_Artifact]) -> list[Path]:
+ """The artifacts that hold no tracked file and, unless they are dependencies, hold only files that git ignores.
+
+ A generated file that the repo commits, or a file git does not ignore, may be
+ someone's work, so a path that holds one stays. A dependency folder may be
+ outside .gitignore, because no one writes their own files into one. When
+ git cannot answer, nothing is approved, and that includes an orphaned
+ worktree, whose lost index leaves git unable to say what its branch tracks.
+ """
- return items
+ if not artifacts:
+ return []
+ gitdir = _gitdir_for(worktree)
+ if gitdir is None or not gitdir.is_dir():
+ click.echo(f" {_display_path(worktree)} is orphaned, so git cannot check it and nothing in it is removed.")
+ return []
+ status = _git(
+ worktree, ["status", "--porcelain=v1", "-z", "--ignored=traditional", "--untracked-files=normal"], timeout=120.0
+ )
+ if status is None or status.returncode != 0:
+ click.echo(f" ⚠️ git could not list what {_display_path(worktree)} ignores, so nothing in it is removed.")
+ return []
+ untracked: set[str] = set()
+ ignored: set[str] = set()
+ entries = iter(status.stdout.split("\0"))
+ for entry in entries:
+ code, path = entry[:2], entry[3:].rstrip("/")
+ if code == "??":
+ untracked.add(path)
+ elif code == "!!":
+ ignored.add(path)
+ elif "R" in code or "C" in code:
+ next(entries, None)
+
+ worktree_resolved = _resolve(worktree)
+ approved: list[Path] = []
+ for artifact in artifacts:
+ if _holds_nested_repo(artifact.path):
+ continue
+ relative = artifact.path.relative_to(worktree_resolved).as_posix()
+ if _is_or_is_under_any(relative, ignored) or (
+ "deps" in artifact.kinds and _is_or_is_under_any(relative, untracked)
+ ):
+ approved.append(artifact.path)
+ return approved
-def _looks_like_cargo_target(path: Path) -> bool:
- return (path / "CACHEDIR.TAG").exists() or (path / "debug").exists() or (path / "release").exists()
+
+def _holds_nested_repo(artifact: Path) -> bool:
+ """Whether a repo that git does not see from outside sits near the top of *artifact*, such as a pip editable install in .venv/src."""
+
+ return artifact.is_dir() and any(
+ True for pattern in ("*/.git", "*/*/.git", "*/*/*/.git") for _ in itertools.islice(artifact.glob(pattern), 1)
+ )
+
+
+def _is_or_is_under_any(relative_path: str, directories: set[str]) -> bool:
+ while relative_path:
+ if relative_path in directories:
+ return True
+ relative_path = relative_path.rpartition("/")[0]
+ return False
def _without_cargo_builds(worktrees: list[Worktree]) -> list[Worktree]:
@@ -1388,7 +1669,7 @@ def _without_cargo_builds(worktrees: list[Worktree]) -> list[Worktree]:
def _cargo_built_since(path: Path, cutoff: float) -> bool:
- return path.name == "target" and _looks_like_cargo_target(path) and last_cargo_build_epoch(path) >= cutoff
+ return path.name == "target" and looks_like_cargo_target(path) and last_cargo_build_epoch(path) >= cutoff
def _clean_legacy_cargo_target(cutoff: float, dry_run: bool, yes: bool) -> None:
@@ -1396,7 +1677,7 @@ def _clean_legacy_cargo_target(cutoff: float, dry_run: bool, yes: bool) -> None:
if target is None:
return
- size = sum(_du_sizes([target]).values())
+ size = measure_disk_usage([target]).freeable_bytes
click.echo(
f"Old shared Cargo target {_display_path(target)}: {_format_size(size)}, "
f"last build {_ago(last_cargo_build_epoch(target))} ago. Cargo built here while the Flox env set CARGO_TARGET_DIR."
@@ -1415,42 +1696,6 @@ def _clean_legacy_cargo_target(cutoff: float, dry_run: bool, yes: bool) -> None:
click.echo()
-def _du_sizes(paths: Sequence[Path]) -> dict[str, float]:
- """On-disk size in bytes per path via a single `du -sk`.
-
- Keyed by the string path passed to `du` (which echoes it back). Parses
- whatever du printed regardless of exit code, so one unreadable path doesn't
- zero the whole batch.
- """
-
- existing = [str(p) for p in paths if p.exists()]
- if not existing:
- return {}
- try:
- result = subprocess.run(
- ["du", "-sk", *existing],
- capture_output=True,
- text=True,
- timeout=_DU_TIMEOUT_SECONDS,
- )
- except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
- return {}
-
- sizes: dict[str, float] = {}
- for line in result.stdout.splitlines():
- if not line.strip():
- continue
- parts = line.split("\t", 1)
- if len(parts) < 2:
- parts = line.split(None, 1)
- if len(parts) < 2:
- continue
- size_kb, path_str = parts[0].strip(), parts[1].strip()
- if size_kb.isdigit():
- sizes[path_str] = int(size_kb) * 1024.0
- return sizes
-
-
def _execute(
worktrees: list[Worktree],
mode: str,
@@ -1464,9 +1709,8 @@ def _execute(
The reclaimed figure is measured from actual filesystem free space before/after,
so it reflects what was genuinely freed — hardlinked (pnpm store) and
- copy-on-write-cloned (APFS) blocks that are shared elsewhere don't count,
- unlike the du-based pre-delete estimate. Falls back to -1.0 if free space
- couldn't be measured.
+ copy-on-write-cloned (APFS) blocks that are shared elsewhere don't count. Falls
+ back to -1.0 if free space couldn't be measured.
"""
# Anchor on each target's parent (which survives the deletion) so disk_usage
@@ -1487,8 +1731,13 @@ def _execute(
click.echo(f" skipped {_display_path(wt.path)}: a running process now has it as its working directory")
continue
before_remove(wt)
- if mode == "deps":
- if _delete_paths(wt.deps_items):
+ if mode != "full":
+ # Git can start tracking a path while the prompt waits, so only paths that git still approves now go.
+ still_approved = set(_git_approved_artifacts(wt.path, _collect_artifacts(wt.path, _artifact_kinds(mode))))
+ items = [item for item in wt.artifact_items if item in still_approved]
+ if not items:
+ continue
+ if _delete_paths(items):
failed += 1
else:
removed += 1
@@ -1585,18 +1834,46 @@ def _rmtree(path: Path) -> None:
def _delete_paths(paths: Sequence[Path]) -> int:
- """Remove deps directories; return how many could not be removed."""
+ """Remove artifact directories and files; return how many could not be removed.
+
+ Every removal outside --mode full goes through here, so the nested-repo
+ guard sits here too. A folder that holds a `.git` entry at any depth may
+ hold commits or edits that no other copy has, and it is kept whatever rule
+ selected it.
+ """
failures = 0
for path in paths:
try:
- _rmtree(path)
+ if path.is_dir() and not path.is_symlink():
+ if _holds_git_entry(path):
+ click.echo(f" ⚠️ kept {_display_path(path)} because a git repository is inside it.")
+ failures += 1
+ continue
+ _rmtree(path)
+ else:
+ path.unlink(missing_ok=True)
except OSError as err:
click.echo(f" ⚠️ could not remove {_display_path(path)}: {err}")
failures += 1
return failures
+def _holds_git_entry(directory: Path) -> bool:
+ pending = [str(directory)]
+ while pending:
+ try:
+ with os.scandir(pending.pop()) as entries:
+ for entry in entries:
+ if entry.name == ".git":
+ return True
+ if entry.is_dir(follow_symlinks=False):
+ pending.append(entry.path)
+ except OSError:
+ return True
+ return False
+
+
def _cleanup_empty_parent(path: Path) -> None:
"""Remove a now-empty parent (e.g. PostHog Desktop's <id>/ after <id>/posthog)."""