Lightweight coding agent that runs in your terminal
Rust
Prune unused standalone releases after successful installation (#55149)
Eric Traut89461ea68338
diff --git a/scripts/install/install.sh b/scripts/install/install.shindex f27d1642be..e40cb02f43 100755--- a/scripts/install/install.sh+++ b/scripts/install/install.sh@@ -214,8 +214,9 @@ parse_release_metadata() {# so the record boundaries inserted by fold do not change the document.LC_ALL=C fold -b -w 4096 | LC_ALL=C awk 'function finish_string(value) {- if (object_depth == 1 && key == "tag_name") {- print "tag_name\t" value+ if (object_depth == 1 &&+ (key == "tag_name" || key == "version" || key == "target" || key == "entrypoint")) {+ print key "\t" value} else if (object_depth == asset_object_depth) {if (key == "name") {asset_name = value@@ -758,6 +759,157 @@ cleanup_stale_install_artifacts() {fi}+record_standalone_releases_in_use() {+ # /proc also catches old clients with no session records. On macOS lsof+ # checks every open file in the release, including running helper programs.+ if [ "$os" = "linux" ]; then+ [ -r /proc/self/mountinfo ] && [ -L /proc/self/exe ] || return 1+ # hidepid can omit other users' processes entirely; do not assume an+ # incomplete process listing proves an older release is unused.+ if grep -E '(^|[ ,])hidepid=[^0, ]' /proc/self/mountinfo >/dev/null; then+ return 1+ fi+ : > "$tmp_dir/in-use" || return 1+ for process in /proc/[0-9]*; do+ if executable="$(readlink "$process/exe" 2>/dev/null)"; then+ printf 'n%s\n' "$executable" >> "$tmp_dir/in-use" || return 1+ elif [ -d "$process" ]; then+ # Even same-user ssh-agent/systemd processes may deny exe inspection.+ # Only an unreadable Codex or bundled helper prevents safe pruning.+ process_name="$(cat "$process/comm" 2>/dev/null)" || {+ [ ! -d "$process" ] && continue+ return 1+ }+ # Linux truncates comm to 15 bytes, including the code-mode host.+ case "$process_name" in+ codex* | rg | bwrap)+ # A zombie has no files left open, even if its parent never reaps it.+ grep -Eq '^State:[[:space:]]*Z' "$process/status" 2>/dev/null && continue+ [ ! -d "$process" ] && continue+ return 1+ ;;+ esac+ fi+ done+ else+ # macOS lsof silently omits other users' processes, even with no errors.+ # ps can still report their executable names. As on Linux, decline to+ # prune when an uninspectable Codex or helper may be using a release.+ current_uid="$(id -u)" || return 1+ ps -axo uid=,comm= > "$tmp_dir/processes" || return 1+ [ -s "$tmp_dir/processes" ] || return 1+ while read -r process_uid process_name; do+ [ "$process_uid" = "$current_uid" ] && continue+ case "${process_name##*/}" in+ codex* | rg | bwrap) return 1 ;;+ esac+ done < "$tmp_dir/processes"+ if /usr/sbin/lsof -nP -F n +D "$releases" > "$tmp_dir/in-use" 2> "$tmp_dir/lsof-error"; then+ :+ else+ # lsof returns 1 for an empty result as well as for errors.+ [ ! -s "$tmp_dir/in-use" ] && [ ! -s "$tmp_dir/lsof-error" ] || return 1+ fi+ [ ! -s "$tmp_dir/lsof-error" ] || return 1+ fi+}++# Run under the install lock, after the visible command has been verified.+# Only retire ordinary, manifest-owned packages; never retry a retired tree.+prune_standalone_releases() (+ [ ! -L "$CODEX_HOME_DIR/packages" ] &&+ [ ! -L "$STANDALONE_ROOT" ] && [ ! -L "$RELEASES_DIR" ] || return 1+ releases="$(cd -P "$RELEASES_DIR" && pwd)" || return 1+ # Local process inspection cannot establish whether a network home is in+ # use on another host. df -l excludes remotely mounted filesystems.+ df -lP "$releases" > /dev/null 2> "$tmp_dir/df-error" || return 1+ [ ! -s "$tmp_dir/df-error" ] || return 1+ selected="$(cd -P "$CURRENT_LINK" && pwd)" || return 1+ history="$(cat "$STANDALONE_ROOT/selection-history")" || return 1+ previous="$(printf '%s\n' "$history" | sed -n '2p')"+ case "$previous" in+ */* | . | ..) return 1 ;;+ esac+ identity="$(LC_ALL=C stat -c '%d %i %.9Z' "$CURRENT_LINK" 2>/dev/null ||+ LC_ALL=C stat -f '%d %i %Fc' "$CURRENT_LINK")" || return 1+ [ "$selected" = "$releases/${selected##*/}" ] &&+ [ "$history" = "$(printf '%s\n%s\n%s' "${selected##*/}" "$previous" "$identity")" ] ||+ return 1+ record_standalone_releases_in_use || return 1+ restore_candidate=""+ # A stopped install must not strand a late-starting release in retirement.+ trap 'if [ -n "$restore_candidate" ] && [ -d "$retired/release" ] &&+ [ ! -e "$candidate" ] && [ ! -L "$candidate" ]; then+ mv "$retired/release" "$candidate" && rmdir "$retired"+ fi' EXIT+ trap 'exit 1' HUP INT TERM++ for candidate in "$releases"/*; do+ [ -d "$candidate" ] && [ ! -L "$candidate" ] &&+ [ "$candidate" != "$selected" ] && [ "${candidate##*/}" != "$previous" ] || continue+ grep -F "n$candidate/" "$tmp_dir/in-use" >/dev/null && continue+ [ -f "$candidate/codex-package.json" ] && [ ! -L "$candidate/codex-package.json" ] || continue+ metadata="$(parse_release_metadata < "$candidate/codex-package.json")" || continue+ version="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "version" {print $2}')"+ target="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "target" {print $2}')"+ entrypoint="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "entrypoint" {print $2}')"+ [ "${candidate##*/}" = "$version-$target" ] && [ "$version" != "latest" ] &&+ [ "$entrypoint" = "bin/codex" ] && [ -f "$candidate/bin/codex" ] || continue+ validate_version "$version" >/dev/null 2>&1 || continue+ case "$target" in+ aarch64-apple-darwin | x86_64-apple-darwin | aarch64-unknown-linux-musl | x86_64-unknown-linux-musl) ;;+ *) continue ;;+ esac++ # Reject symlinks (except the installer's own alias), special files and+ # mounts before rm -r. find -xdev alone would not protect nested mounts.+ [ "$(readlink "$candidate/codex" 2>/dev/null)" = "bin/codex" ] || continue+ if ! find "$candidate" -xdev ! -type f ! -type d ! -path "$candidate/codex" -print > "$tmp_dir/unowned"; then+ continue+ fi+ [ ! -s "$tmp_dir/unowned" ] || continue+ if [ "$os" = "linux" ]; then+ # mountinfo escapes spaces, tabs, newlines, and backslashes in paths.+ if ! CODEX_PRUNE_RELEASE="$candidate" LC_ALL=C awk '+ BEGIN { release = ENVIRON["CODEX_PRUNE_RELEASE"] }+ { path = $5; gsub(/\\040/, " ", path); gsub(/\\011/, "\t", path)+ gsub(/\\012/, "\n", path); gsub(/\\134/, "\\", path)+ if (path == release || index(path, release "/") == 1) found = 1 }+ END { exit found ? 1 : 0 }+ ' /proc/self/mountinfo; then+ continue+ fi+ else+ device="$(stat -f '%d' "$releases")" || return 1+ if ! find "$candidate" -xdev -exec stat -f '%d' {} + > "$tmp_dir/devices"; then+ continue+ fi+ grep -vxF "$device" "$tmp_dir/devices" >/dev/null && continue+ fi++ retired="$(mktemp -d "$releases/.retired.XXXXXXXX")" || return 1+ restore_candidate=1+ if mv "$candidate" "$retired/release"; then+ # Block new launches at the original path before the final use check.+ # Some systems report the old executable path, so check both names.+ if ! record_standalone_releases_in_use ||+ grep -F -e "n$candidate/" -e "n$retired/release/" "$tmp_dir/in-use" >/dev/null; then+ [ ! -e "$candidate" ] && [ ! -L "$candidate" ] || return 1+ mv "$retired/release" "$candidate" && rmdir "$retired" || return 1+ restore_candidate=""+ continue+ fi+ # Once proved unused, never restore a possibly partially deleted tree.+ restore_candidate=""+ find "$retired" -xdev -depth -delete || return 1+ else+ restore_candidate=""+ rmdir "$retired"+ return 1+ fi+ done+)+replace_path_with_symlink() {link_path="$1"link_target="$2"@@ -1267,6 +1419,8 @@ if [ "$DAEMON_ONLY" = "1" ] && [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ];echo "The production release does not support daemon-owned packages; the current selection was left unchanged." >&2exit 1fi+fi+if [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ]; then# A no-op must not replace the selection link: history is bound to it.selected_dir="$(cd -P "$release_dir" && pwd)"previous_dir="$(cd -P "$CURRENT_LINK" 2>/dev/null && pwd)" || previous_dir=""@@ -1280,7 +1434,7 @@ if [ "$DAEMON_ONLY" = "1" ] && [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ];printf '%s\n%s\n%s\n' "$release_name" "${previous_dir##*/}" "$selection_id" > "$history.tmp.$$"mv -f "$history.tmp.$$" "$history"); then- warn "Could not record daemon selection; pruning will be skipped."+ warn "Could not record installation selection; pruning will be skipped."fifielse@@ -1300,6 +1454,9 @@ fiupdate_visible_command "$release_dir"add_to_pathverify_visible_command+if ! prune_standalone_releases; then+ warn "Could not safely inspect old Codex releases; skipped remaining cleanup."+firelease_install_lockhandle_conflicting_installdiff --git a/scripts/install/test_install_sh.py b/scripts/install/test_install_sh.pyindex 2d495cddaf..ad69ac4b78 100644--- a/scripts/install/test_install_sh.py+++ b/scripts/install/test_install_sh.py@@ -1,9 +1,13 @@#!/usr/bin/env python3+from concurrent.futures import ThreadPoolExecutorimport hashlibimport jsonimport osfrom pathlib import Path+import select+import shlex+import shutilimport subprocessimport tarfileimport tempfile@@ -317,6 +321,224 @@ class InstallShTest(unittest.TestCase):self.assertEqual(guarded.returncode, 0, guarded.stderr)self.assertTrue(managed.exists())+ def test_standalone_retains_previous_and_running_releases_across_installs(+ self,+ ) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ # Exercise real process inspection, including paths containing spaces.+ root = Path(temp_dir).resolve() / "codex install"+ root.mkdir()+ system = os.uname()+ if system.sysname == "Linux":+ # The race requires pruning to proceed. A live but unreadable+ # Codex/helper (e.g. the agent's sandbox) must prevent that.+ for process in Path("/proc").glob("[0-9]*"):+ try:+ (process / "exe").readlink()+ except OSError:+ try:+ name = (process / "comm").read_text().strip()+ status = (process / "status").read_text()+ except OSError:+ continue # The process may have exited.+ if (+ name.startswith("codex") or name in ("rg", "bwrap")+ ) and not any(+ line.startswith("State:") and "Z" in line+ for line in status.splitlines()+ ):+ self.skipTest(+ f"Cannot inspect live {name} process {process.name}"+ )+ arch = "aarch64" if system.machine in ("arm64", "aarch64") else "x86_64"+ target = f"{arch}-{'apple-darwin' if system.sysname == 'Darwin' else 'unknown-linux-musl'}"+ packages = root / "codex-home/packages/standalone"+ releases = packages / "releases"+ versions = ("0.140.0", "0.141.0", VERSION)+ install_options = {}+ for version in versions:+ source = root / version+ source.mkdir()+ archive, checksum, metadata = create_package_release(+ source,+ metadata_version=version,+ binary_version=version,+ target=target,+ )+ install_options[version] = dict(+ metadata_json=metadata, archive_path=archive, checksum_path=checksum+ )++ first, _ = run_installer_in(+ root, versions[0], **install_options[versions[0]]+ )+ self.assertEqual(first.returncode, 0, first.stderr)+ oldest = releases / f"{versions[0]}-{target}"+ helper = oldest / "bin/codex-code-mode-host"+ shutil.copy("/bin/sleep", helper)+ if system.sysname == "Darwin":+ # A copied macOS system binary cannot run with its platform signature.+ subprocess.run(+ ["/usr/bin/codesign", "--force", "--sign", "-", str(helper)],+ check=True,+ capture_output=True,+ )+ # A helper can outlive its parent CLI; retain its whole release.+ with subprocess.Popen([str(helper), "300"]) as running:+ try:+ second, _ = run_installer_in(+ root, versions[1], **install_options[versions[1]]+ )+ self.assertEqual(second.returncode, 0, second.stderr)++ unowned = releases / f"0.139.0-{target}"+ unowned.mkdir()+ (unowned / "codex-package.json").write_text("{}")+ retired = releases / ".retired.old-attempt"+ retired.mkdir()+ (retired / "leave-alone").write_text("not retried")+ external = root / "external"+ external.mkdir()+ (external / "user-file").write_text("untouched")+ (releases / f"0.138.0-{target}").symlink_to(external)++ # Failed installs must not advance history or prune anything.+ history = packages / "selection-history"+ before_failure = history.read_bytes()+ bad_source = root / "wrong-version"+ bad_source.mkdir()+ archive, checksum, metadata = create_package_release(+ bad_source, metadata_version="0.199.0", target=target+ )+ failed, _ = run_installer_in(+ root,+ "0.199.0",+ metadata_json=metadata,+ archive_path=archive,+ checksum_path=checksum,+ )+ self.assertNotEqual(failed.returncode, 0, failed.stderr)+ self.assertEqual(history.read_bytes(), before_failure)+ self.assertTrue(oldest.is_dir())++ latest, _ = run_installer_in(+ root, "latest", **install_options[VERSION]+ )+ self.assertEqual(latest.returncode, 0, latest.stderr)+ self.assertIsNone(running.poll())+ self.assertTrue(oldest.is_dir(), latest.stderr)+ previous = releases / f"{versions[1]}-{target}"+ current = releases / f"{VERSION}-{target}"+ self.assertEqual((packages / "current").resolve(), current)+ self.assertTrue(previous.is_dir())+ self.assertEqual(+ history.read_text().splitlines()[:2],+ [current.name, previous.name],+ )+ recorded = history.read_bytes()+ finally:+ running.terminate()+ running.wait()++ if system.sysname == "Darwin":+ # lsof sees no running helper now. ps can still see names for+ # a different user, whose open files lsof silently omits.+ fake_ps = root / "bin/ps"+ foreign_uid = 0 if os.geteuid() != 0 else 501+ try:+ for executable in (oldest / "bin/codex", helper):+ with self.subTest(foreign_executable=executable.name):+ fake_ps.write_text(+ "#!/bin/sh\n"+ 'case "$*" in\n'+ " '-axo uid=,comm=')\n"+ f" printf '%s\\n' {shlex.quote(f'{foreign_uid} {executable}')}\n"+ " ;;\n"+ ' *) exec /bin/ps "$@" ;;\n'+ "esac\n"+ )+ fake_ps.chmod(0o755)+ retained, _ = run_installer_in(+ root, "latest", **install_options[VERSION]+ )+ self.assertEqual(retained.returncode, 0, retained.stderr)+ self.assertTrue(oldest.is_dir(), retained.stderr)+ self.assertEqual(history.read_bytes(), recorded)+ finally:+ fake_ps.unlink()++ # Start the helper after the first scan but immediately before the+ # retirement rename. The second scan must restore its release.+ fake_mv = root / "bin/mv"+ ready = root / "retirement-ready"+ resume = root / "retirement-resume"+ os.mkfifo(ready)+ os.mkfifo(resume)+ real_mv = shutil.which("mv")+ fake_mv.write_text(+ "#!/bin/sh\n"+ 'case "$2" in\n'+ " */.retired.*/release)\n"+ f' printf "ready\\n" > {shlex.quote(str(ready))}\n'+ f" read -r _ < {shlex.quote(str(resume))}\n"+ " ;;\n"+ "esac\n"+ f'exec {shlex.quote(real_mv)} "$@"\n'+ )+ fake_mv.chmod(0o755)+ ready_fd = os.open(ready, os.O_RDWR | os.O_NONBLOCK)+ resume_fd = os.open(resume, os.O_RDWR | os.O_NONBLOCK)+ try:+ with ThreadPoolExecutor(max_workers=1) as executor:+ installing = executor.submit(+ run_installer_in, root, "latest", **install_options[VERSION]+ )+ try:+ readable, _, _ = select.select([ready_fd], [], [], 30)+ self.assertTrue(+ readable, "installer did not attempt retirement"+ )+ self.assertEqual(os.read(ready_fd, 128), b"ready\n")+ with subprocess.Popen([str(helper), "300"]) as late_helper:+ try:+ os.write(resume_fd, b"continue\n")+ raced, _ = installing.result(timeout=30)+ self.assertEqual(raced.returncode, 0, raced.stderr)+ self.assertTrue(oldest.is_dir(), raced.stderr)+ self.assertEqual(history.read_bytes(), recorded)+ finally:+ late_helper.terminate()+ late_helper.wait()+ finally:+ # Release the intercepted mv even if an assertion fails.+ os.write(resume_fd, b"continue\n")+ finally:+ os.close(ready_fd)+ os.close(resume_fd)+ fake_mv.unlink()++ # A no-op latest install (also the codex update path) cleans up the+ # now-unused release without rotating previous to current.+ repeated, _ = run_installer_in(root, "latest", **install_options[VERSION])+ self.assertEqual(repeated.returncode, 0, repeated.stderr)+ self.assertFalse(oldest.exists(), repeated.stderr)+ self.assertTrue(previous.is_dir())+ self.assertEqual(history.read_bytes(), recorded)+ self.assertTrue(unowned.is_dir())+ self.assertEqual((retired / "leave-alone").read_text(), "not retried")+ self.assertEqual((external / "user-file").read_text(), "untouched")++ # Downgrades keep the actual previously selected version.+ pinned, _ = run_installer_in(+ root, versions[1], **install_options[versions[1]]+ )+ self.assertEqual(pinned.returncode, 0, pinned.stderr)+ self.assertEqual((packages / "current").resolve(), previous)+ self.assertEqual(+ history.read_text().splitlines()[:2], [previous.name, current.name]+ )+ self.assertTrue(current.is_dir())+def test_uninspectable_legacy_updater_does_not_clear_pin(self) -> None:with tempfile.TemporaryDirectory() as temp_dir:root = Path(temp_dir)@@ -937,22 +1159,32 @@ def create_package_release(root: Path,*,metadata_version: str = VERSION,+ binary_version: str = VERSION,+ target: str = "aarch64-apple-darwin",) -> tuple[Path, Path, str]:package_dir = root / "package"(package_dir / "bin").mkdir(parents=True)(package_dir / "codex-path").mkdir()- (package_dir / "codex-package.json").write_text("{}\n", encoding="utf-8")+ (package_dir / "codex-package.json").write_text(+ json.dumps(+ {"version": binary_version, "target": target, "entrypoint": "bin/codex"}+ ),+ encoding="utf-8",+ )write_executable(package_dir / "bin" / "codex",- f"#!/bin/sh\nprintf 'codex-cli {VERSION}\\n'\n",+ f"#!/bin/sh\nprintf 'codex-cli {binary_version}\\n'\n",)write_executable(package_dir / "bin" / "codex-code-mode-host","#!/bin/sh\nexit 0\n",)write_executable(package_dir / "codex-path" / "rg", "#!/bin/sh\nexit 0\n")+ if "linux" in target:+ (package_dir / "codex-resources").mkdir()+ write_executable(package_dir / "codex-resources/bwrap", "#!/bin/sh\nexit 0\n")- asset = "codex-package-aarch64-apple-darwin.tar.gz"+ asset = f"codex-package-{target}.tar.gz"archive_path = root / assetwith tarfile.open(archive_path, "w:gz") as archive:for path in package_dir.iterdir():