Skip to content

The agent that grows with you

fix(git): graph refresh honours global core.commitGraph, serializes its slot and covers linked worktrees

Teknium25db2df10308
diff --git a/hermes_cli/_subprocess_compat.py b/hermes_cli/_subprocess_compat.py
index cb65bb85c8c..596279d1fc9 100644
--- a/hermes_cli/_subprocess_compat.py
+++ b/hermes_cli/_subprocess_compat.py
@@ -739,7 +739,8 @@ def kill_process_tree(proc: subprocess.Popen, *, grace: float = 0.0) -> None:
"""Best-effort terminate *proc* and its descendants on both platforms; never raises.
``grace`` > 0 (POSIX): SIGTERM the tree first and SIGKILL only if *proc* outlives ``grace``
- seconds, so a git can delete its lock and tmp files on the way out (Windows always forces).
+ seconds, so a git can delete its lock and tmp files on the way out (Windows always forces);
+ when *proc* exits in time, what is left of the group it leads is SIGKILLed.
``proc.kill()`` alone only terminates the direct child. This is cleanup on an already-failing
path whose contract is to fail open, so every failure (access denied, already reaped) is
@@ -762,12 +763,24 @@ def kill_process_tree(proc: subprocess.Popen, *, grace: float = 0.0) -> None:
if grace > 0 and not IS_WINDOWS:
import signal as _signal
+ try:
+ leads_group = os.getpgid(proc.pid) == proc.pid
+ except ProcessLookupError:
+ leads_group = False
_deadline_kill_tree(proc.pid, sig=_signal.SIGTERM)
try:
proc.wait(timeout=grace)
- return
except subprocess.TimeoutExpired:
pass
+ else:
+ # The leader went on TERM; a group member that ignores it would outlive us. The
+ # pgid stays reserved while any member lives, so this reaches only that group.
+ if leads_group:
+ try:
+ os.killpg(proc.pid, _signal.SIGKILL) # windows-footgun: ok — POSIX-only branch
+ except ProcessLookupError:
+ pass
+ return
_deadline_kill_tree(proc.pid)
except Exception:
_legacy_kill_process_tree(proc)
diff --git a/hermes_cli/git_commit_graph.py b/hermes_cli/git_commit_graph.py
index 850124c1267..c8dd4f7f83a 100644
--- a/hermes_cli/git_commit_graph.py
+++ b/hermes_cli/git_commit_graph.py
@@ -20,8 +20,9 @@ import subprocess
import sys
import tempfile
import time
+from contextlib import contextmanager
from pathlib import Path
-from typing import Optional, Sequence
+from typing import Iterator, Optional, Sequence
from hermes_cli._subprocess_compat import (
IS_WINDOWS,
@@ -50,34 +51,60 @@ _CHILD = ("import sys; sys.path.insert(0, sys.argv[1]); "
"from hermes_cli.git_commit_graph import refresh_in_child; refresh_in_child(*sys.argv[2:])")
+@contextmanager
+def _locked_slot(lock: Path, *, wait: bool) -> Iterator[Optional[int]]:
+ """The slot file's fd under an exclusive cross-process lock, or None while another process holds
+ it (``wait=False``). The file is never unlinked: a waiter on a deleted inode would claim a slot
+ nobody else can see. The kernel drops the lock with the process."""
+ fd = os.open(lock, os.O_RDWR | os.O_CREAT, 0o644)
+ try:
+ try:
+ if IS_WINDOWS:
+ import msvcrt
+
+ msvcrt.locking(fd, msvcrt.LK_LOCK if wait else msvcrt.LK_NBLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(fd, fcntl.LOCK_EX | (0 if wait else fcntl.LOCK_NB))
+ except OSError:
+ yield None
+ return
+ try:
+ yield fd
+ finally:
+ if IS_WINDOWS:
+ os.lseek(fd, 0, os.SEEK_SET)
+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
+ finally:
+ os.close(fd)
+
+
def claim_maintenance_slot(git_dir: Path, lock_name: str, interval: float) -> bool:
"""Exactly one process per clone gets to run a maintenance job per ``interval``.
- The lock file's mtime is the stamp: younger than the interval means another launch is
- running the job (or just tried and timed out) — skip. A stale lock is taken over by ``replace``,
- which only one of N racing processes can win; the O_EXCL create then serializes against a
- process that found no lock at all.
+ The slot file holds the claimant's pid and its mtime is the stamp: younger than the interval
+ means another launch is running the job (or just tried and timed out) — skip. Check and claim
+ happen under one file lock, so of N launches finding an expired or empty slot exactly one wins.
"""
- lock = git_dir / lock_name
- try:
- st = lock.stat()
- except FileNotFoundError:
- pass
- else:
- if time.time() - st.st_mtime < interval:
+ with _locked_slot(git_dir / lock_name, wait=False) as fd:
+ if fd is None:
return False
- try:
- lock.replace(lock.with_suffix(".stale"))
- except OSError:
+ st = os.fstat(fd)
+ if st.st_size and time.time() - st.st_mtime < interval:
return False
- try:
- fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
- except FileExistsError:
- return False
- with os.fdopen(fd, "w", encoding="utf-8") as fh:
- fh.write(f"{os.getpid()}\n")
- lock.with_suffix(".stale").unlink(missing_ok=True)
- return True
+ os.ftruncate(fd, 0)
+ os.lseek(fd, 0, os.SEEK_SET)
+ os.write(fd, f"{os.getpid()}\n".encode())
+ return True
+
+
+def release_maintenance_slot(git_dir: Path, lock_name: str) -> None:
+ """Give back this process's claim (its job never started) so the next run retries; a claim
+ another process made since is left alone."""
+ with _locked_slot(git_dir / lock_name, wait=True) as fd:
+ if fd is not None and os.read(fd, 32).strip() == str(os.getpid()).encode():
+ os.ftruncate(fd, 0)
def run_bounded_git(repo_root: str, args: Sequence[str], timeout: float, env: Optional[dict] = None) -> Optional[int]:
@@ -108,7 +135,7 @@ def run_bounded_git(repo_root: str, args: Sequence[str], timeout: float, env: Op
rc = proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
_reap()
- logger.info("git %s exceeded %ds; killed", args[0], timeout)
+ logger.warning("git %s in %s exceeded %ss; killed", args[0], repo_root, timeout)
return None
finally:
atexit.unregister(_reap)
@@ -138,41 +165,74 @@ def claim_commit_graph_refresh(git_dir: Path) -> bool:
def _commit_graph_disabled(repo_root: str) -> bool:
- """``core.commitGraph=false``: the owner turned graphs off in this repo; never write one there."""
+ """``core.commitGraph=false`` in the repo, global or system config: never write a graph there.
+
+ Read with the user's global/system config, which the writer's isolated env blanks."""
+ env = noninteractive_git_env()
+ for key in ("GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM"):
+ env.pop(key)
+ if key in os.environ:
+ env[key] = os.environ[key]
probe = bounded_probe_run(["git", "config", "--type=bool", "--get", "core.commitGraph"],
- timeout=10, cwd=repo_root, env=noninteractive_git_env())
+ timeout=10, cwd=repo_root, env=env)
return probe is not None and probe.stdout.strip() == "false"
-def refresh_commit_graph(repo_root: str) -> None:
- """Clear what a killed writer stranded, then write the graph (bounded, niced, never lazy-fetching)."""
- from hermes_cli.gitlock import OBJECT_STORE_LOCK_NAMES, clear_stale_git_locks, clear_stale_tmp_packs
+def common_git_dir(repo_root: str | Path) -> Optional[Path]:
+ """The git dir holding the object store: ``.git`` itself, or the dir a linked worktree's or
+ submodule's ``.git`` file leads to. None when ``repo_root`` is no checkout."""
+ from hermes_cli.update_lock import _git_common_dir
+
+ return _git_common_dir(Path(repo_root))
+
+
+def refresh_commit_graph(repo_root: str, git_dir: Path) -> None:
+ """Clear what a killed writer stranded, then write the graph (bounded, niced, never lazy-fetching).
+
+ ``git_dir`` is the common git dir (:func:`common_git_dir`): objects, debris and stamp live there."""
+ from hermes_cli.gitlock import clear_stale_object_store_locks, clear_stale_tmp_packs
if _commit_graph_disabled(repo_root):
logger.info("core.commitGraph=false in %s; not writing a commit-graph", repo_root)
return
- root = Path(repo_root)
# Object-store debris only: ``hermes -w`` runs this in whatever repo it was launched from, where
# index.lock / HEAD.lock belong to the user's own git (and pgrep can be blind to it).
- clear_stale_git_locks(root, names=OBJECT_STORE_LOCK_NAMES)
- clear_stale_tmp_packs(root)
+ clear_stale_object_store_locks(git_dir)
+ clear_stale_tmp_packs(git_dir)
logger.info("git commit-graph missing or stale — writing it in background")
if run_bounded_git(repo_root, _WRITE, _TIMEOUT, env={**noninteractive_git_env(), **NO_LAZY_FETCH_ENV}) == 0:
- (root / ".git" / _FRESH_STAMP).touch()
+ (git_dir / _FRESH_STAMP).touch()
+
+
+class _LogDirOnFailureHandler(logging.FileHandler):
+ """Opens (``delay``) only when there is a failure to record, creating ``logs/`` then: a home
+ whose logs were never set up still gets the line; a deleted profile is not resurrected. A log
+ that cannot be opened drops the line (``handleError``) instead of raising out of the logger."""
+
+ def emit(self, record: logging.LogRecord) -> None:
+ if self.stream is None:
+ from hermes_constants import mkdir_under_hermes_home
+
+ try:
+ mkdir_under_hermes_home(Path(self.baseFilename).parent)
+ self.stream = self._open()
+ except OSError: # a missing/deleted named profile is FileNotFoundError
+ self.handleError(record)
+ return
+ super().emit(record)
-def refresh_in_child(repo_root: str, log_file: str) -> None:
+def refresh_in_child(repo_root: str, git_dir: str, log_file: str) -> None:
"""Body of the detached child: a failure is appended to the home's agent.log; never raises.
A plain handler, not ``setup_logging``: that reads config.yaml and seeds the home's layout,
side effects a git maintenance child has no business having in a profile mid-update.
- ``delay`` opens the log only when there is a failure to record.
"""
- handler = logging.FileHandler(log_file, encoding="utf-8", delay=True)
+ handler = _LogDirOnFailureHandler(log_file, encoding="utf-8", delay=True)
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
logging.getLogger().addHandler(handler)
try:
- refresh_commit_graph(repo_root)
+ refresh_commit_graph(repo_root, Path(git_dir))
except Exception:
logger.warning("Background commit-graph refresh of %s failed", repo_root, exc_info=True)
@@ -187,11 +247,12 @@ def spawn_commit_graph_refresh(repo_root: Path) -> bool:
"""
from hermes_constants import get_hermes_home
- git_dir = Path(repo_root) / ".git"
- if not (git_dir / "objects").is_dir() or not claim_commit_graph_refresh(git_dir):
+ git_dir = common_git_dir(repo_root)
+ if git_dir is None or not (git_dir / "objects").is_dir() or not claim_commit_graph_refresh(git_dir):
return False
code_root = Path(__file__).resolve().parents[1]
- argv = [sys.executable, "-c", _CHILD, str(code_root), str(repo_root), str(get_hermes_home() / "logs" / "agent.log")]
+ argv = [sys.executable, "-c", _CHILD, str(code_root), str(repo_root), str(git_dir),
+ str(get_hermes_home() / "logs" / "agent.log")]
kwargs: dict = dict(cwd=str(repo_root), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, close_fds=True)
try:
@@ -203,6 +264,6 @@ def spawn_commit_graph_refresh(repo_root: Path) -> bool:
# A job object without BREAKAWAY_OK refuses CREATE_BREAKAWAY_FROM_JOB; retry without it.
subprocess.Popen(argv, **kwargs, creationflags=windows_detach_flags_without_breakaway())
except OSError:
- (git_dir / _LOCK).unlink(missing_ok=True)
+ release_maintenance_slot(git_dir, _LOCK)
raise
return True
diff --git a/hermes_cli/gitlock.py b/hermes_cli/gitlock.py
index cb126a16771..a6fc500bb35 100644
--- a/hermes_cli/gitlock.py
+++ b/hermes_cli/gitlock.py
@@ -103,15 +103,7 @@ def _sweep_stale(directory: Path, candidates: Callable[[], Iterable[Path]], *, m
return removed
-def clear_stale_git_locks(repo_root: Path, *, min_age_seconds: Optional[int] = None,
- names: Iterable[str] = (*LOCK_NAMES, *OBJECT_STORE_LOCK_NAMES)) -> list[str]:
- """Remove abandoned ``.git`` lock files under ``repo_root``; returns the removed paths.
-
- Removes only when older than the age floor AND no git process is running. Never raises: a lock we cannot
- stat/unlink is skipped (it may have been re-created between the age check and the unlink; skipping is safe).
- ``names`` narrows the sweep (relative to ``.git``) for repos Hermes does not own.
- """
- git_dir = Path(repo_root) / ".git"
+def _sweep_locks(git_dir: Path, names: Iterable[str], min_age_seconds: Optional[int]) -> list[str]:
return _sweep_stale(
git_dir, lambda: [git_dir / name for name in names],
min_age_seconds=min_age_seconds, default_age=STALE_LOCK_MIN_AGE_SECONDS,
@@ -120,6 +112,21 @@ def clear_stale_git_locks(repo_root: Path, *, min_age_seconds: Optional[int] = N
)
+def clear_stale_git_locks(repo_root: Path, *, min_age_seconds: Optional[int] = None) -> list[str]:
+ """Remove abandoned ``.git`` lock files under ``repo_root``; returns the removed paths.
+
+ Removes only when older than the age floor AND no git process is running. Never raises: a lock we cannot
+ stat/unlink is skipped (it may have been re-created between the age check and the unlink; skipping is safe).
+ """
+ return _sweep_locks(Path(repo_root) / ".git", (*LOCK_NAMES, *OBJECT_STORE_LOCK_NAMES), min_age_seconds)
+
+
+def clear_stale_object_store_locks(git_dir: Path) -> list[str]:
+ """``OBJECT_STORE_LOCK_NAMES`` only, under ``git_dir`` itself (a linked worktree's or submodule's
+ common dir), for repos Hermes does not own: index.lock / HEAD.lock there are the user's git's."""
+ return _sweep_locks(Path(git_dir), OBJECT_STORE_LOCK_NAMES, None)
+
+
def release_dead_index_lock(repo_root: Path) -> bool:
"""Drop ``.git/index.lock`` when the git that took it is proven gone, whatever its age.
diff --git a/hermes_cli/worktree_ops.py b/hermes_cli/worktree_ops.py
index ede73e02676..bc932825b73 100644
--- a/hermes_cli/worktree_ops.py
+++ b/hermes_cli/worktree_ops.py
@@ -22,6 +22,7 @@ from hermes_cli._subprocess_compat import noninteractive_git_env
from hermes_cli.git_commit_graph import (
claim_commit_graph_refresh,
claim_maintenance_slot,
+ common_git_dir,
refresh_commit_graph,
run_bounded_git,
)
@@ -147,21 +148,22 @@ def _maintain_pack_health(repo_root: str) -> None:
pack index and worktree creation can blow its timeout under concurrent load.
"""
try:
- pack_dir = Path(repo_root) / ".git" / "objects" / "pack"
- if not pack_dir.is_dir():
+ # A linked worktree's or submodule's ``.git`` is a file; its objects live in the common dir.
+ git_dir = common_git_dir(repo_root)
+ pack_dir = git_dir / "objects" / "pack" if git_dir else None
+ if pack_dir is None or not pack_dir.is_dir():
return
- git_dir = pack_dir.parent.parent
packs = len(list(pack_dir.glob("*.pack")))
if packs >= _PACK_SPRAWL_THRESHOLD and _claim_repack_slot(git_dir):
from hermes_cli.gitlock import clear_stale_tmp_packs
- clear_stale_tmp_packs(Path(repo_root))
+ clear_stale_tmp_packs(git_dir)
logger.info("git pack sprawl (%d packs) — repacking in background", packs)
_run_bounded_repack(repo_root)
# Repacking can strand now-duplicated admin files; prune on the same pass.
_git(["worktree", "prune"], repo_root, timeout=60, check=False,
stdin=subprocess.DEVNULL, env=noninteractive_git_env())
if claim_commit_graph_refresh(git_dir):
- refresh_commit_graph(repo_root)
+ refresh_commit_graph(repo_root, git_dir)
except Exception as e:
logger.debug("pack maintenance skipped: %s", e)
diff --git a/tests/hermes_cli/test_git_probe_tree_kill.py b/tests/hermes_cli/test_git_probe_tree_kill.py
index a308c8dd69d..34aaa7736a3 100644
--- a/tests/hermes_cli/test_git_probe_tree_kill.py
+++ b/tests/hermes_cli/test_git_probe_tree_kill.py
@@ -112,6 +112,28 @@ def test_group_kill_skipped_when_child_shares_our_group():
assert proc.returncode is not None
+def test_grace_kill_takes_a_term_ignoring_member_once_the_leader_exits(tmp_path):
+ """``grace`` returns as soon as the leader exits on SIGTERM; a group member that ignores
+ SIGTERM must not outlive it (the leader-only wait used to orphan it)."""
+ marker = tmp_path / "member.pid"
+ script = tmp_path / "leader.sh"
+ script.write_text(f"#!/usr/bin/env bash\n(trap '' TERM; echo $BASHPID > {marker}; exec sleep 300) &\nsleep 300\n")
+ script.chmod(0o755)
+ proc = subprocess.Popen([str(script)], process_group=0, stdin=subprocess.DEVNULL)
+ member = _wait_marker(marker)
+
+ kill_process_tree(proc, grace=5.0)
+
+ assert proc.returncode is not None
+ deadline = time.monotonic() + 3.0
+ while time.monotonic() < deadline and _pid_alive(member):
+ time.sleep(0.05)
+ alive = _pid_alive(member)
+ if alive: # reparented out of our subtree, so the live-system guard refuses os.kill here
+ subprocess.run(["kill", "-9", str(member)], check=False)
+ assert not alive, f"SIGTERM-ignoring member {member} survived the graced kill"
+
+
def test_fast_path_unaffected(tmp_path):
"""Successful probes behave exactly as before the group-kill port."""
subprocess.run(["git", "init", "-q", str(tmp_path / "repo")], check=True)
diff --git a/tests/hermes_cli/test_gitlock.py b/tests/hermes_cli/test_gitlock.py
index b6cd5454905..60940b3faa9 100644
--- a/tests/hermes_cli/test_gitlock.py
+++ b/tests/hermes_cli/test_gitlock.py
@@ -103,7 +103,7 @@ def test_graph_refresh_clears_object_store_debris_and_leaves_the_index_lock(repo
for path in (*debris, git_dir / "index.lock"):
path.parent.mkdir(parents=True, exist_ok=True)
_touch(path, STALE_LOCK_MIN_AGE_SECONDS + 60)
- refresh_commit_graph(str(repo))
+ refresh_commit_graph(str(repo), git_dir)
assert [path for path in debris if path.exists()] == []
assert (git_dir / "index.lock").exists()
subprocess.run(["git", "commit-graph", "verify"], cwd=repo, check=True)
diff --git a/tests/hermes_cli/test_worktree_selfheal.py b/tests/hermes_cli/test_worktree_selfheal.py
index 1bf41ab0ba2..98f467b4280 100644
--- a/tests/hermes_cli/test_worktree_selfheal.py
+++ b/tests/hermes_cli/test_worktree_selfheal.py
@@ -146,7 +146,8 @@ class TestMaintainPackHealth:
def test_pass_refreshes_a_stale_graph_in_a_treeless_clone_without_fetching(self, tmp_path):
"""A graph that stopped covering HEAD makes every ``git tag --merged HEAD`` walk the gap. In a
partial clone over a Bloom graph, a write that recomputes changed paths lazy-fetches the tree
- of every new commit (#127711): the refresh must write the graph and download nothing."""
+ of every new commit (#127711): the refresh must write the graph and download nothing. Run from
+ a linked worktree, whose ``.git`` is a file: the object store is the clone's."""
import os
import time
@@ -177,11 +178,14 @@ class TestMaintainPackHealth:
old = time.time() - 3 * 86400
for graph in (info / "commit-graphs").iterdir():
os.utime(graph, (old, old))
+ linked = tmp_path / "linked"
+ _git(clone, "worktree", "add", "-q", "--no-checkout", "--detach", str(linked), "HEAD")
packs = sorted((clone / ".git" / "objects" / "pack").glob("*.pack"))
- worktree_ops._maintain_pack_health(str(clone))
+ worktree_ops._maintain_pack_health(str(linked))
assert (info / "commit-graphs" / "commit-graph-chain").stat().st_mtime > time.time() - 300
+ assert (clone / ".git" / "hermes-commit-graph.fresh").is_file()
assert sorted((clone / ".git" / "objects" / "pack").glob("*.pack")) == packs, "the write lazy-fetched"
_git(clone, "commit-graph", "verify")
diff --git a/website/docs/getting-started/updating.md b/website/docs/getting-started/updating.md
index 6b94e8ed51e..30a65740b49 100644
--- a/website/docs/getting-started/updating.md
+++ b/website/docs/getting-started/updating.md
@@ -318,8 +318,10 @@ whole history: each `hermes update` (after the new code is in place) and each `h
update a missing or day-old graph in the background, at most once every 6 hours, with
`git commit-graph write --reachable --split --no-changed-paths`, which reads commits only and never
downloads a tree. The update does not wait for it, and a write still running after 10 minutes is
-stopped (asked to exit first, so git removes its own lock). A failed write is logged to
-`logs/agent.log`. A repo with `core.commitGraph=false` is left without a graph. A
+stopped (asked to exit first, so git removes its own lock). A failed or stopped write is logged to
+`logs/agent.log`. A checkout that is a linked worktree or submodule refreshes the graph of the
+repository it shares objects with. A repo with `core.commitGraph=false` in its own, your global or
+the system git config is left without a graph. A
`commit-graph.lock` or `multi-pack-index.lock` that a killed git left behind is removed by the same
stale-lock sweep as `index.lock` once it is 10 minutes old and no git is running; in a repo you
launch `hermes -w` from, only those object-store locks are swept, never `index.lock` or `HEAD.lock`.