Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Run exec-server compatibility tests under Bazel (#40736)

Adam Perry @ OpenAI62aacbb2c9a7
diff --git a/bazel/rules/testing/compat/BUILD.bazel b/bazel/rules/testing/compat/BUILD.bazel
new file mode 100644
index 0000000000..9f4bf9b0ba
--- /dev/null
+++ b/bazel/rules/testing/compat/BUILD.bazel
@@ -0,0 +1,31 @@
+load("@rules_rust//rust:defs.bzl", "rust_test")
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["exec_server_compat_test.bzl"])
+
+# exec_server_compat_test wraps this shared binary with current or released builds.
+# Keep it manual so only wrapped compatibility tests enter the normal test graph.
+rust_test(
+ name = "exec-server-compat-test-bin",
+ srcs = ["exec_server_compat_test.rs"],
+ crate_name = "exec_server_compat_test",
+ crate_root = "exec_server_compat_test.rs",
+ edition = "2024",
+ tags = ["manual"],
+ target_compatible_with = [
+ "@platforms//cpu:x86_64",
+ "@platforms//os:linux",
+ ],
+ visibility = ["//codex-rs/exec-server:__pkg__"],
+ deps = [
+ "//codex-rs/exec-server",
+ "//codex-rs/exec-server/tests/support",
+ "@crates//:anyhow",
+ "@crates//:pretty_assertions",
+ "@crates//:serde_json",
+ "@crates//:tempfile",
+ "@crates//:tokio",
+ "@crates//:wiremock",
+ ],
+)
diff --git a/bazel/rules/testing/compat/exec_server_compat_test.bzl b/bazel/rules/testing/compat/exec_server_compat_test.bzl
new file mode 100644
index 0000000000..8e9f6c7533
--- /dev/null
+++ b/bazel/rules/testing/compat/exec_server_compat_test.bzl
@@ -0,0 +1,47 @@
+"""Bazel coverage for app-server and exec-server release compatibility."""
+
+load("//:defs.bzl", "workspace_root_test")
+
+def exec_server_compat_test(
+ name,
+ comparison_binary = None,
+ current_binary = "//codex-rs/cli:codex",
+ release = None):
+ """Tests both executor directions against another Codex build or release.
+
+ Args:
+ name: Name of the generated compatibility test target.
+ comparison_binary: Built Codex executable to compare with the current build.
+ current_binary: Built Codex executable representing the current version.
+ release: External release repository exposing `:codex` and `:package`.
+ """
+ if (comparison_binary == None) == (release == None):
+ fail("exactly one of comparison_binary and release must be set")
+
+ comparison = comparison_binary if release == None else release + "//:codex"
+ data = [] if release == None else [release + "//:package"]
+ comparison_alias = name + "-comparison-binary"
+ native.alias(
+ name = comparison_alias,
+ actual = comparison,
+ tags = ["manual"],
+ visibility = ["//visibility:private"],
+ )
+
+ workspace_root_test(
+ name = name,
+ args = ["--test-threads=1"],
+ data = data,
+ runfile_env = {
+ "//codex-rs/bwrap:bwrap": "CARGO_BIN_EXE_bwrap",
+ current_binary: "CODEX_TEST_CURRENT_CODEX",
+ ":" + comparison_alias: "CODEX_TEST_RELEASED_CODEX",
+ },
+ tags = ["no-sandbox"],
+ target_compatible_with = [
+ "@platforms//cpu:x86_64",
+ "@platforms//os:linux",
+ ],
+ test_bin = "//bazel/rules/testing/compat:exec-server-compat-test-bin",
+ workspace_root_marker = "//codex-rs/utils/cargo-bin:repo_root.marker",
+ )
diff --git a/codex-rs/exec-server/tests/relay/version_skew.rs b/bazel/rules/testing/compat/exec_server_compat_test.rs
similarity index 95%
rename from codex-rs/exec-server/tests/relay/version_skew.rs
rename to bazel/rules/testing/compat/exec_server_compat_test.rs
index 7543ae8c54..863d5a85e1 100644
--- a/codex-rs/exec-server/tests/relay/version_skew.rs
+++ b/bazel/rules/testing/compat/exec_server_compat_test.rs
@@ -3,7 +3,6 @@ use std::path::PathBuf;
use std::process::Stdio;
use std::sync::Arc;
use std::sync::Mutex;
-use std::time::Duration;
use anyhow::Context;
use anyhow::Result;
@@ -11,6 +10,11 @@ use codex_exec_server::CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN_ENV_VAR;
use codex_exec_server::CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID_ENV_VAR;
use codex_exec_server::CODEX_EXEC_SERVER_NOISE_REGISTRY_URL_ENV_VAR;
use codex_exec_server::CODEX_EXEC_SERVER_URL_ENV_VAR;
+use codex_exec_server_test_support::relay::TEST_TIMEOUT as VERSION_SKEW_TIMEOUT;
+use codex_exec_server_test_support::relay::accept_websocket;
+use codex_exec_server_test_support::relay::assert_relay_data_is_encrypted;
+use codex_exec_server_test_support::relay::proxy_relay_frames;
+use codex_exec_server_test_support::relay::registered_executor_public_key;
use pretty_assertions::assert_eq;
use serde_json::Value;
use serde_json::json;
@@ -31,19 +35,13 @@ use wiremock::matchers::header;
use wiremock::matchers::method;
use wiremock::matchers::path;
-use crate::relay_support::ENVIRONMENT_ID;
-use crate::relay_support::EXECUTOR_REGISTRATION_ID;
-use crate::relay_support::HARNESS_KEY_AUTHORIZATION;
-use crate::relay_support::REGISTRY_TOKEN;
-use crate::relay_support::accept_websocket;
-use crate::relay_support::assert_relay_data_is_encrypted;
-use crate::relay_support::proxy_relay_frames;
-use crate::relay_support::registered_executor_public_key;
-
+const ENVIRONMENT_ID: &str = "env-noise-relay-test";
+const EXECUTOR_REGISTRATION_ID: &str = "registration-1";
+const HARNESS_KEY_AUTHORIZATION: &str = "harness-key-authorization";
+const REGISTRY_TOKEN: &str = "registry-token";
const RELEASED_CODEX_ENV_VAR: &str = "CODEX_TEST_RELEASED_CODEX";
const CURRENT_CODEX_ENV_VAR: &str = "CODEX_TEST_CURRENT_CODEX";
const EXECUTOR_MARKER_ENV_VAR: &str = "CODEX_EXECUTOR_VERSION_SKEW_MARKER";
-const VERSION_SKEW_TIMEOUT: Duration = Duration::from_secs(30);
const EXPECTED_OUTPUT: &str = "executor-version-skew-ok";
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
diff --git a/codex-rs/exec-server-protocol/src/lib.rs b/codex-rs/exec-server-protocol/src/lib.rs
index bf6a057fe9..bdfaa3d145 100644
--- a/codex-rs/exec-server-protocol/src/lib.rs
+++ b/codex-rs/exec-server-protocol/src/lib.rs
@@ -9,6 +9,3 @@ pub use network_policy::*;
pub use process_id::ProcessId;
pub use protocol::*;
pub use rpc::*;
-
-/// Oldest Codex release supported by the executor protocol.
-pub const MINIMUM_SUPPORTED_CODEX_VERSION: &str = "0.145.0";
diff --git a/codex-rs/exec-server/BUILD.bazel b/codex-rs/exec-server/BUILD.bazel
index d70b389698..d894273ced 100644
--- a/codex-rs/exec-server/BUILD.bazel
+++ b/codex-rs/exec-server/BUILD.bazel
@@ -1,4 +1,5 @@
load("//:defs.bzl", "codex_rust_crate")
+load("//bazel/rules/testing/compat:exec_server_compat_test.bzl", "exec_server_compat_test")
exports_files(
["src/proto/codex.exec_server.relay.v1.rs"],
@@ -26,3 +27,19 @@ codex_rust_crate(
test_tags = ["no-sandbox"],
unit_test_args = ["--test-threads=1"],
)
+
+exec_server_compat_test(
+ name = "exec-server-current-version-test",
+ comparison_binary = "//codex-rs/cli:codex",
+ current_binary = "//codex-rs/cli:codex",
+)
+
+exec_server_compat_test(
+ name = "exec-server-stable-release-test",
+ release = "@codex_release_0.149.1_linux_x86_64",
+)
+
+exec_server_compat_test(
+ name = "exec-server-minimum-release-test",
+ release = "@codex_release_0.145.0_linux_x86_64",
+)
diff --git a/codex-rs/exec-server/testing/run_version_skew.sh b/codex-rs/exec-server/testing/run_version_skew.sh
deleted file mode 100755
index dd52dc63eb..0000000000
--- a/codex-rs/exec-server/testing/run_version_skew.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
-minimum_supported_release="$(
- sed -n 's/^pub const MINIMUM_SUPPORTED_CODEX_VERSION: &str = "\([^"]*\)";$/\1/p' \
- "${repo_root}/codex-rs/exec-server-protocol/src/lib.rs"
-)"
-: "${minimum_supported_release:?minimum supported Codex release is missing}"
-release_directory="$(mktemp -d "${TMPDIR:-/tmp}/codex-exec-server-skew.XXXXXX")"
-trap 'rm -rf "${release_directory:?}"' EXIT
-
-if [[ $# -eq 0 ]]; then
- releases=(latest "${minimum_supported_release}")
-else
- releases=("$@")
-fi
-
-case "$(uname -s):$(uname -m)" in
- Darwin:arm64) target="aarch64-apple-darwin" ;;
- Darwin:x86_64) target="x86_64-apple-darwin" ;;
- Linux:aarch64 | Linux:arm64) target="aarch64-unknown-linux-musl" ;;
- Linux:x86_64) target="x86_64-unknown-linux-musl" ;;
- *)
- echo "Unsupported platform: $(uname -s) $(uname -m)" >&2
- exit 1
- ;;
-esac
-
-asset="codex-${target}.tar.gz"
-
-if [[ "$(uname -s)" == "Linux" ]] \
- && command -v apt-get >/dev/null \
- && command -v dpkg-deb >/dev/null \
- && command -v dpkg-architecture >/dev/null \
- && ! pkg-config --exists alsa glib-2.0 gobject-2.0 gio-2.0; then
- # Debian-like hosts can download development metadata without sudo. The
- # temporary sysroot still relies on runtime libraries installed on the host.
- voice_package_directory="${release_directory}/voice-packages"
- voice_sysroot="${release_directory}/voice-sysroot"
- mkdir -p "${voice_package_directory}" "${voice_sysroot}"
- (
- cd "${voice_package_directory}"
- apt-get download libasound2-dev libglib2.0-dev libcap-dev libffi-dev libmount-dev \
- libpcre2-dev libselinux1-dev zlib1g-dev libblkid-dev libsepol-dev
- )
- for package in "${voice_package_directory}"/*.deb; do
- dpkg-deb --extract "${package}" "${voice_sysroot}"
- done
-
- voice_multiarch="$(dpkg-architecture -qDEB_HOST_MULTIARCH)"
- voice_library_directory="${voice_sysroot}/usr/lib/${voice_multiarch}"
- for library in "${voice_library_directory}"/*.so; do
- [[ -L "${library}" ]] || continue
- library_target="$(readlink "${library}")"
- if [[ ! -e "${voice_library_directory}/${library_target}" && -e "/usr/lib/${voice_multiarch}/${library_target}" ]]; then
- ln -s "/usr/lib/${voice_multiarch}/${library_target}" "${voice_library_directory}/${library_target}"
- fi
- done
-
- export PKG_CONFIG_PATH="${voice_library_directory}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}"
- export PKG_CONFIG_SYSROOT_DIR="${voice_sysroot}"
-fi
-
-cd "${repo_root}/codex-rs"
-cargo build -p codex-cli --bin codex
-export CODEX_TEST_CURRENT_CODEX="${CARGO_TARGET_DIR:-${repo_root}/codex-rs/target}/debug/codex"
-
-echo "Testing current Codex compatibility through authenticated Noise"
-export CODEX_TEST_RELEASED_CODEX="${CODEX_TEST_CURRENT_CODEX}"
-just test -p codex-exec-server --test relay version_skew --test-threads 1
-
-tested_release_version=""
-for release in "${releases[@]}"; do
- release="${release#rust-v}"
- if [[ "${release}" == "${tested_release_version}" ]]; then
- echo "Skipping Codex ${release}; this release was already tested"
- continue
- fi
-
- if [[ "${release}" == "latest" ]]; then
- release_url="https://github.com/openai/codex/releases/latest/download/${asset}"
- else
- release_url="https://github.com/openai/codex/releases/download/rust-v${release}/${asset}"
- fi
-
- binary_directory="${release_directory}/${release}"
- mkdir -p "${binary_directory}"
- echo "Downloading released Codex from ${release_url}"
- curl -fsSL "${release_url}" -o "${binary_directory}/${asset}"
- tar -xzf "${binary_directory}/${asset}" -C "${binary_directory}"
-
- export CODEX_TEST_RELEASED_CODEX="${binary_directory}/codex-${target}"
- release_output="$("${CODEX_TEST_RELEASED_CODEX}" --version)"
- echo "${release_output}"
- tested_release_version="${release_output##* }"
-
- just test -p codex-exec-server --test relay version_skew --test-threads 1
-done
diff --git a/codex-rs/exec-server/testing/run_version_skew_test.sh b/codex-rs/exec-server/testing/run_version_skew_test.sh
deleted file mode 100755
index 109a61cefa..0000000000
--- a/codex-rs/exec-server/testing/run_version_skew_test.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-script_path="${script_directory}/run_version_skew.sh"
-fake_bin="$(mktemp -d "${TMPDIR:-/tmp}/codex-version-skew-test.XXXXXX")"
-trap 'rm -rf "${fake_bin:?}"' EXIT
-
-for tool in bash dirname mkdir mktemp rm sed; do
- ln -s "$(command -v "${tool}")" "${fake_bin}/${tool}"
-done
-
-printf '%s\n' \
- '#!/usr/bin/env bash' \
- 'case "${1:-}" in' \
- ' -s) printf "Linux\\n" ;;' \
- ' -m) printf "x86_64\\n" ;;' \
- ' *) exit 1 ;;' \
- 'esac' >"${fake_bin}/uname"
-chmod +x "${fake_bin}/uname"
-
-printf '%s\n' \
- '#!/usr/bin/env bash' \
- 'exit 1' >"${fake_bin}/pkg-config"
-chmod +x "${fake_bin}/pkg-config"
-
-printf '%s\n' \
- '#!/usr/bin/env bash' \
- 'printf "reached cargo\\n" >&2' \
- 'exit 97' >"${fake_bin}/cargo"
-chmod +x "${fake_bin}/cargo"
-
-set +e
-output="$(PATH="${fake_bin}" "${script_path}" latest 2>&1)"
-status=$?
-set -e
-
-if [[ "${status}" -ne 97 || "${output}" != *"reached cargo"* ]]; then
- printf '%s\n' "expected non-Debian Linux without voice metadata to reach cargo" >&2
- printf 'status: %s\n%s\n' "${status}" "${output}" >&2
- exit 1
-fi
diff --git a/codex-rs/exec-server/tests/relay.rs b/codex-rs/exec-server/tests/relay.rs
index 3201538366..f7dc181ed6 100644
--- a/codex-rs/exec-server/tests/relay.rs
+++ b/codex-rs/exec-server/tests/relay.rs
@@ -1,9 +1,5 @@
mod common;
-#[cfg(unix)]
-#[path = "relay/version_skew.rs"]
-mod version_skew;
-
#[path = "common/relay.rs"]
mod relay_support;