Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Prune unused standalone releases after successful installation (#55149)

Eric Traut89461ea68338
diff --git a/scripts/install/install.sh b/scripts/install/install.sh
index f27d1642be..e40cb02f43 100755
--- a/scripts/install/install.sh
+++ b/scripts/install/install.sh
@@ -214,8 +214,9 @@ parse_release_metadata() {
# so the record boundaries inserted by fold do not change the document.
LC_ALL=C fold -b -w 4096 | LC_ALL=C awk '
function finish_string(value) {
- if (object_depth == 1 && key == "tag_name") {
- print "tag_name\t" value
+ if (object_depth == 1 &&
+ (key == "tag_name" || key == "version" || key == "target" || key == "entrypoint")) {
+ print key "\t" value
} else if (object_depth == asset_object_depth) {
if (key == "name") {
asset_name = value
@@ -758,6 +759,157 @@ cleanup_stale_install_artifacts() {
fi
}
+record_standalone_releases_in_use() {
+ # /proc also catches old clients with no session records. On macOS lsof
+ # checks every open file in the release, including running helper programs.
+ if [ "$os" = "linux" ]; then
+ [ -r /proc/self/mountinfo ] && [ -L /proc/self/exe ] || return 1
+ # hidepid can omit other users' processes entirely; do not assume an
+ # incomplete process listing proves an older release is unused.
+ if grep -E '(^|[ ,])hidepid=[^0, ]' /proc/self/mountinfo >/dev/null; then
+ return 1
+ fi
+ : > "$tmp_dir/in-use" || return 1
+ for process in /proc/[0-9]*; do
+ if executable="$(readlink "$process/exe" 2>/dev/null)"; then
+ printf 'n%s\n' "$executable" >> "$tmp_dir/in-use" || return 1
+ elif [ -d "$process" ]; then
+ # Even same-user ssh-agent/systemd processes may deny exe inspection.
+ # Only an unreadable Codex or bundled helper prevents safe pruning.
+ process_name="$(cat "$process/comm" 2>/dev/null)" || {
+ [ ! -d "$process" ] && continue
+ return 1
+ }
+ # Linux truncates comm to 15 bytes, including the code-mode host.
+ case "$process_name" in
+ codex* | rg | bwrap)
+ # A zombie has no files left open, even if its parent never reaps it.
+ grep -Eq '^State:[[:space:]]*Z' "$process/status" 2>/dev/null && continue
+ [ ! -d "$process" ] && continue
+ return 1
+ ;;
+ esac
+ fi
+ done
+ else
+ # macOS lsof silently omits other users' processes, even with no errors.
+ # ps can still report their executable names. As on Linux, decline to
+ # prune when an uninspectable Codex or helper may be using a release.
+ current_uid="$(id -u)" || return 1
+ ps -axo uid=,comm= > "$tmp_dir/processes" || return 1
+ [ -s "$tmp_dir/processes" ] || return 1
+ while read -r process_uid process_name; do
+ [ "$process_uid" = "$current_uid" ] && continue
+ case "${process_name##*/}" in
+ codex* | rg | bwrap) return 1 ;;
+ esac
+ done < "$tmp_dir/processes"
+ if /usr/sbin/lsof -nP -F n +D "$releases" > "$tmp_dir/in-use" 2> "$tmp_dir/lsof-error"; then
+ :
+ else
+ # lsof returns 1 for an empty result as well as for errors.
+ [ ! -s "$tmp_dir/in-use" ] && [ ! -s "$tmp_dir/lsof-error" ] || return 1
+ fi
+ [ ! -s "$tmp_dir/lsof-error" ] || return 1
+ fi
+}
+
+# Run under the install lock, after the visible command has been verified.
+# Only retire ordinary, manifest-owned packages; never retry a retired tree.
+prune_standalone_releases() (
+ [ ! -L "$CODEX_HOME_DIR/packages" ] &&
+ [ ! -L "$STANDALONE_ROOT" ] && [ ! -L "$RELEASES_DIR" ] || return 1
+ releases="$(cd -P "$RELEASES_DIR" && pwd)" || return 1
+ # Local process inspection cannot establish whether a network home is in
+ # use on another host. df -l excludes remotely mounted filesystems.
+ df -lP "$releases" > /dev/null 2> "$tmp_dir/df-error" || return 1
+ [ ! -s "$tmp_dir/df-error" ] || return 1
+ selected="$(cd -P "$CURRENT_LINK" && pwd)" || return 1
+ history="$(cat "$STANDALONE_ROOT/selection-history")" || return 1
+ previous="$(printf '%s\n' "$history" | sed -n '2p')"
+ case "$previous" in
+ */* | . | ..) return 1 ;;
+ esac
+ identity="$(LC_ALL=C stat -c '%d %i %.9Z' "$CURRENT_LINK" 2>/dev/null ||
+ LC_ALL=C stat -f '%d %i %Fc' "$CURRENT_LINK")" || return 1
+ [ "$selected" = "$releases/${selected##*/}" ] &&
+ [ "$history" = "$(printf '%s\n%s\n%s' "${selected##*/}" "$previous" "$identity")" ] ||
+ return 1
+ record_standalone_releases_in_use || return 1
+ restore_candidate=""
+ # A stopped install must not strand a late-starting release in retirement.
+ trap 'if [ -n "$restore_candidate" ] && [ -d "$retired/release" ] &&
+ [ ! -e "$candidate" ] && [ ! -L "$candidate" ]; then
+ mv "$retired/release" "$candidate" && rmdir "$retired"
+ fi' EXIT
+ trap 'exit 1' HUP INT TERM
+
+ for candidate in "$releases"/*; do
+ [ -d "$candidate" ] && [ ! -L "$candidate" ] &&
+ [ "$candidate" != "$selected" ] && [ "${candidate##*/}" != "$previous" ] || continue
+ grep -F "n$candidate/" "$tmp_dir/in-use" >/dev/null && continue
+ [ -f "$candidate/codex-package.json" ] && [ ! -L "$candidate/codex-package.json" ] || continue
+ metadata="$(parse_release_metadata < "$candidate/codex-package.json")" || continue
+ version="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "version" {print $2}')"
+ target="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "target" {print $2}')"
+ entrypoint="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "entrypoint" {print $2}')"
+ [ "${candidate##*/}" = "$version-$target" ] && [ "$version" != "latest" ] &&
+ [ "$entrypoint" = "bin/codex" ] && [ -f "$candidate/bin/codex" ] || continue
+ validate_version "$version" >/dev/null 2>&1 || continue
+ case "$target" in
+ aarch64-apple-darwin | x86_64-apple-darwin | aarch64-unknown-linux-musl | x86_64-unknown-linux-musl) ;;
+ *) continue ;;
+ esac
+
+ # Reject symlinks (except the installer's own alias), special files and
+ # mounts before rm -r. find -xdev alone would not protect nested mounts.
+ [ "$(readlink "$candidate/codex" 2>/dev/null)" = "bin/codex" ] || continue
+ if ! find "$candidate" -xdev ! -type f ! -type d ! -path "$candidate/codex" -print > "$tmp_dir/unowned"; then
+ continue
+ fi
+ [ ! -s "$tmp_dir/unowned" ] || continue
+ if [ "$os" = "linux" ]; then
+ # mountinfo escapes spaces, tabs, newlines, and backslashes in paths.
+ if ! CODEX_PRUNE_RELEASE="$candidate" LC_ALL=C awk '
+ BEGIN { release = ENVIRON["CODEX_PRUNE_RELEASE"] }
+ { path = $5; gsub(/\\040/, " ", path); gsub(/\\011/, "\t", path)
+ gsub(/\\012/, "\n", path); gsub(/\\134/, "\\", path)
+ if (path == release || index(path, release "/") == 1) found = 1 }
+ END { exit found ? 1 : 0 }
+ ' /proc/self/mountinfo; then
+ continue
+ fi
+ else
+ device="$(stat -f '%d' "$releases")" || return 1
+ if ! find "$candidate" -xdev -exec stat -f '%d' {} + > "$tmp_dir/devices"; then
+ continue
+ fi
+ grep -vxF "$device" "$tmp_dir/devices" >/dev/null && continue
+ fi
+
+ retired="$(mktemp -d "$releases/.retired.XXXXXXXX")" || return 1
+ restore_candidate=1
+ if mv "$candidate" "$retired/release"; then
+ # Block new launches at the original path before the final use check.
+ # Some systems report the old executable path, so check both names.
+ if ! record_standalone_releases_in_use ||
+ grep -F -e "n$candidate/" -e "n$retired/release/" "$tmp_dir/in-use" >/dev/null; then
+ [ ! -e "$candidate" ] && [ ! -L "$candidate" ] || return 1
+ mv "$retired/release" "$candidate" && rmdir "$retired" || return 1
+ restore_candidate=""
+ continue
+ fi
+ # Once proved unused, never restore a possibly partially deleted tree.
+ restore_candidate=""
+ find "$retired" -xdev -depth -delete || return 1
+ else
+ restore_candidate=""
+ rmdir "$retired"
+ return 1
+ fi
+ done
+)
+
replace_path_with_symlink() {
link_path="$1"
link_target="$2"
@@ -1267,6 +1419,8 @@ if [ "$DAEMON_ONLY" = "1" ] && [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ];
echo "The production release does not support daemon-owned packages; the current selection was left unchanged." >&2
exit 1
fi
+fi
+if [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ]; then
# A no-op must not replace the selection link: history is bound to it.
selected_dir="$(cd -P "$release_dir" && pwd)"
previous_dir="$(cd -P "$CURRENT_LINK" 2>/dev/null && pwd)" || previous_dir=""
@@ -1280,7 +1434,7 @@ if [ "$DAEMON_ONLY" = "1" ] && [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ];
printf '%s\n%s\n%s\n' "$release_name" "${previous_dir##*/}" "$selection_id" > "$history.tmp.$$"
mv -f "$history.tmp.$$" "$history"
); then
- warn "Could not record daemon selection; pruning will be skipped."
+ warn "Could not record installation selection; pruning will be skipped."
fi
fi
else
@@ -1300,6 +1454,9 @@ fi
update_visible_command "$release_dir"
add_to_path
verify_visible_command
+if ! prune_standalone_releases; then
+ warn "Could not safely inspect old Codex releases; skipped remaining cleanup."
+fi
release_install_lock
handle_conflicting_install
diff --git a/scripts/install/test_install_sh.py b/scripts/install/test_install_sh.py
index 2d495cddaf..ad69ac4b78 100644
--- a/scripts/install/test_install_sh.py
+++ b/scripts/install/test_install_sh.py
@@ -1,9 +1,13 @@
#!/usr/bin/env python3
+from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
+import select
+import shlex
+import shutil
import subprocess
import tarfile
import tempfile
@@ -317,6 +321,224 @@ class InstallShTest(unittest.TestCase):
self.assertEqual(guarded.returncode, 0, guarded.stderr)
self.assertTrue(managed.exists())
+ def test_standalone_retains_previous_and_running_releases_across_installs(
+ self,
+ ) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Exercise real process inspection, including paths containing spaces.
+ root = Path(temp_dir).resolve() / "codex install"
+ root.mkdir()
+ system = os.uname()
+ if system.sysname == "Linux":
+ # The race requires pruning to proceed. A live but unreadable
+ # Codex/helper (e.g. the agent's sandbox) must prevent that.
+ for process in Path("/proc").glob("[0-9]*"):
+ try:
+ (process / "exe").readlink()
+ except OSError:
+ try:
+ name = (process / "comm").read_text().strip()
+ status = (process / "status").read_text()
+ except OSError:
+ continue # The process may have exited.
+ if (
+ name.startswith("codex") or name in ("rg", "bwrap")
+ ) and not any(
+ line.startswith("State:") and "Z" in line
+ for line in status.splitlines()
+ ):
+ self.skipTest(
+ f"Cannot inspect live {name} process {process.name}"
+ )
+ arch = "aarch64" if system.machine in ("arm64", "aarch64") else "x86_64"
+ target = f"{arch}-{'apple-darwin' if system.sysname == 'Darwin' else 'unknown-linux-musl'}"
+ packages = root / "codex-home/packages/standalone"
+ releases = packages / "releases"
+ versions = ("0.140.0", "0.141.0", VERSION)
+ install_options = {}
+ for version in versions:
+ source = root / version
+ source.mkdir()
+ archive, checksum, metadata = create_package_release(
+ source,
+ metadata_version=version,
+ binary_version=version,
+ target=target,
+ )
+ install_options[version] = dict(
+ metadata_json=metadata, archive_path=archive, checksum_path=checksum
+ )
+
+ first, _ = run_installer_in(
+ root, versions[0], **install_options[versions[0]]
+ )
+ self.assertEqual(first.returncode, 0, first.stderr)
+ oldest = releases / f"{versions[0]}-{target}"
+ helper = oldest / "bin/codex-code-mode-host"
+ shutil.copy("/bin/sleep", helper)
+ if system.sysname == "Darwin":
+ # A copied macOS system binary cannot run with its platform signature.
+ subprocess.run(
+ ["/usr/bin/codesign", "--force", "--sign", "-", str(helper)],
+ check=True,
+ capture_output=True,
+ )
+ # A helper can outlive its parent CLI; retain its whole release.
+ with subprocess.Popen([str(helper), "300"]) as running:
+ try:
+ second, _ = run_installer_in(
+ root, versions[1], **install_options[versions[1]]
+ )
+ self.assertEqual(second.returncode, 0, second.stderr)
+
+ unowned = releases / f"0.139.0-{target}"
+ unowned.mkdir()
+ (unowned / "codex-package.json").write_text("{}")
+ retired = releases / ".retired.old-attempt"
+ retired.mkdir()
+ (retired / "leave-alone").write_text("not retried")
+ external = root / "external"
+ external.mkdir()
+ (external / "user-file").write_text("untouched")
+ (releases / f"0.138.0-{target}").symlink_to(external)
+
+ # Failed installs must not advance history or prune anything.
+ history = packages / "selection-history"
+ before_failure = history.read_bytes()
+ bad_source = root / "wrong-version"
+ bad_source.mkdir()
+ archive, checksum, metadata = create_package_release(
+ bad_source, metadata_version="0.199.0", target=target
+ )
+ failed, _ = run_installer_in(
+ root,
+ "0.199.0",
+ metadata_json=metadata,
+ archive_path=archive,
+ checksum_path=checksum,
+ )
+ self.assertNotEqual(failed.returncode, 0, failed.stderr)
+ self.assertEqual(history.read_bytes(), before_failure)
+ self.assertTrue(oldest.is_dir())
+
+ latest, _ = run_installer_in(
+ root, "latest", **install_options[VERSION]
+ )
+ self.assertEqual(latest.returncode, 0, latest.stderr)
+ self.assertIsNone(running.poll())
+ self.assertTrue(oldest.is_dir(), latest.stderr)
+ previous = releases / f"{versions[1]}-{target}"
+ current = releases / f"{VERSION}-{target}"
+ self.assertEqual((packages / "current").resolve(), current)
+ self.assertTrue(previous.is_dir())
+ self.assertEqual(
+ history.read_text().splitlines()[:2],
+ [current.name, previous.name],
+ )
+ recorded = history.read_bytes()
+ finally:
+ running.terminate()
+ running.wait()
+
+ if system.sysname == "Darwin":
+ # lsof sees no running helper now. ps can still see names for
+ # a different user, whose open files lsof silently omits.
+ fake_ps = root / "bin/ps"
+ foreign_uid = 0 if os.geteuid() != 0 else 501
+ try:
+ for executable in (oldest / "bin/codex", helper):
+ with self.subTest(foreign_executable=executable.name):
+ fake_ps.write_text(
+ "#!/bin/sh\n"
+ 'case "$*" in\n'
+ " '-axo uid=,comm=')\n"
+ f" printf '%s\\n' {shlex.quote(f'{foreign_uid} {executable}')}\n"
+ " ;;\n"
+ ' *) exec /bin/ps "$@" ;;\n'
+ "esac\n"
+ )
+ fake_ps.chmod(0o755)
+ retained, _ = run_installer_in(
+ root, "latest", **install_options[VERSION]
+ )
+ self.assertEqual(retained.returncode, 0, retained.stderr)
+ self.assertTrue(oldest.is_dir(), retained.stderr)
+ self.assertEqual(history.read_bytes(), recorded)
+ finally:
+ fake_ps.unlink()
+
+ # Start the helper after the first scan but immediately before the
+ # retirement rename. The second scan must restore its release.
+ fake_mv = root / "bin/mv"
+ ready = root / "retirement-ready"
+ resume = root / "retirement-resume"
+ os.mkfifo(ready)
+ os.mkfifo(resume)
+ real_mv = shutil.which("mv")
+ fake_mv.write_text(
+ "#!/bin/sh\n"
+ 'case "$2" in\n'
+ " */.retired.*/release)\n"
+ f' printf "ready\\n" > {shlex.quote(str(ready))}\n'
+ f" read -r _ < {shlex.quote(str(resume))}\n"
+ " ;;\n"
+ "esac\n"
+ f'exec {shlex.quote(real_mv)} "$@"\n'
+ )
+ fake_mv.chmod(0o755)
+ ready_fd = os.open(ready, os.O_RDWR | os.O_NONBLOCK)
+ resume_fd = os.open(resume, os.O_RDWR | os.O_NONBLOCK)
+ try:
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ installing = executor.submit(
+ run_installer_in, root, "latest", **install_options[VERSION]
+ )
+ try:
+ readable, _, _ = select.select([ready_fd], [], [], 30)
+ self.assertTrue(
+ readable, "installer did not attempt retirement"
+ )
+ self.assertEqual(os.read(ready_fd, 128), b"ready\n")
+ with subprocess.Popen([str(helper), "300"]) as late_helper:
+ try:
+ os.write(resume_fd, b"continue\n")
+ raced, _ = installing.result(timeout=30)
+ self.assertEqual(raced.returncode, 0, raced.stderr)
+ self.assertTrue(oldest.is_dir(), raced.stderr)
+ self.assertEqual(history.read_bytes(), recorded)
+ finally:
+ late_helper.terminate()
+ late_helper.wait()
+ finally:
+ # Release the intercepted mv even if an assertion fails.
+ os.write(resume_fd, b"continue\n")
+ finally:
+ os.close(ready_fd)
+ os.close(resume_fd)
+ fake_mv.unlink()
+
+ # A no-op latest install (also the codex update path) cleans up the
+ # now-unused release without rotating previous to current.
+ repeated, _ = run_installer_in(root, "latest", **install_options[VERSION])
+ self.assertEqual(repeated.returncode, 0, repeated.stderr)
+ self.assertFalse(oldest.exists(), repeated.stderr)
+ self.assertTrue(previous.is_dir())
+ self.assertEqual(history.read_bytes(), recorded)
+ self.assertTrue(unowned.is_dir())
+ self.assertEqual((retired / "leave-alone").read_text(), "not retried")
+ self.assertEqual((external / "user-file").read_text(), "untouched")
+
+ # Downgrades keep the actual previously selected version.
+ pinned, _ = run_installer_in(
+ root, versions[1], **install_options[versions[1]]
+ )
+ self.assertEqual(pinned.returncode, 0, pinned.stderr)
+ self.assertEqual((packages / "current").resolve(), previous)
+ self.assertEqual(
+ history.read_text().splitlines()[:2], [previous.name, current.name]
+ )
+ self.assertTrue(current.is_dir())
+
def test_uninspectable_legacy_updater_does_not_clear_pin(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
@@ -937,22 +1159,32 @@ def create_package_release(
root: Path,
*,
metadata_version: str = VERSION,
+ binary_version: str = VERSION,
+ target: str = "aarch64-apple-darwin",
) -> tuple[Path, Path, str]:
package_dir = root / "package"
(package_dir / "bin").mkdir(parents=True)
(package_dir / "codex-path").mkdir()
- (package_dir / "codex-package.json").write_text("{}\n", encoding="utf-8")
+ (package_dir / "codex-package.json").write_text(
+ json.dumps(
+ {"version": binary_version, "target": target, "entrypoint": "bin/codex"}
+ ),
+ encoding="utf-8",
+ )
write_executable(
package_dir / "bin" / "codex",
- f"#!/bin/sh\nprintf 'codex-cli {VERSION}\\n'\n",
+ f"#!/bin/sh\nprintf 'codex-cli {binary_version}\\n'\n",
)
write_executable(
package_dir / "bin" / "codex-code-mode-host",
"#!/bin/sh\nexit 0\n",
)
write_executable(package_dir / "codex-path" / "rg", "#!/bin/sh\nexit 0\n")
+ if "linux" in target:
+ (package_dir / "codex-resources").mkdir()
+ write_executable(package_dir / "codex-resources/bwrap", "#!/bin/sh\nexit 0\n")
- asset = "codex-package-aarch64-apple-darwin.tar.gz"
+ asset = f"codex-package-{target}.tar.gz"
archive_path = root / asset
with tarfile.open(archive_path, "w:gz") as archive:
for path in package_dir.iterdir():