Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Stage Python runtime wheels directly from package directories (#45526)

Charlie Marshb0d95427c244
diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml
index 7b529f8cf7..27e8b96458 100644
--- a/.github/workflows/rust-release-windows.yml
+++ b/.github/workflows/rust-release-windows.yml
@@ -535,23 +535,18 @@ jobs:
# The wheel's Windows support floor predates the native voice
# runtime's verified device/VC++ prerequisites. Keep it voice-free.
- wheel_archives="${RUNNER_TEMP}/voice-free-wheel/${{ matrix.target }}"
- bash "${GITHUB_WORKSPACE}/.github/scripts/build-codex-package-archive.sh" \
- --target "${{ matrix.target }}" --bundle primary \
- --entrypoint-dir "$CARGO_TARGET_DIR/${{ matrix.target }}/release" \
- --archive-dir "$wheel_archives"
- python - "$wheel_archives/codex-package-${{ matrix.target }}.tar.gz" <<'PY'
+ wheel_package="${RUNNER_TEMP}/codex-package-${{ matrix.target }}"
+ python - "$wheel_package" <<'PY'
import sys
- import tarfile
- with tarfile.open(sys.argv[1]) as archive:
- assert not any("codex-resources/voice/" in member.name for member in archive)
+ from pathlib import Path
+ assert not (Path(sys.argv[1]) / "codex-resources" / "voice").exists()
PY
stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${{ matrix.target }}"
wheel_dir="${GITHUB_WORKSPACE}/python-runtime-dist/${{ matrix.target }}"
python "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" \
stage-runtime \
"$stage_dir" \
- "$wheel_archives/codex-package-${{ matrix.target }}.tar.gz" \
+ "$wheel_package" \
--codex-version "${GITHUB_REF_NAME}" \
--platform-tag "$platform_tag"
"${RUNNER_TEMP}/python-runtime-build-venv/Scripts/python.exe" -m build --wheel --outdir "$wheel_dir" "$stage_dir"
diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py
index 0c97d5156e..68234bfc88 100755
--- a/sdk/python/scripts/update_sdk_artifacts.py
+++ b/sdk/python/scripts/update_sdk_artifacts.py
@@ -188,9 +188,18 @@ def stage_python_sdk_package(
def stage_python_runtime_package(
staging_dir: Path,
codex_version: str,
- package_archive: Path,
+ package_source: Path,
platform_tag: str | None = None,
) -> Path:
+ if package_source.is_dir():
+ source = package_source.resolve()
+ destination = staging_dir.resolve()
+ if source.is_relative_to(destination) or destination.is_relative_to(source):
+ raise RuntimeError("Codex package and runtime staging directories must not overlap")
+ for path in package_source.rglob("*"):
+ if path.is_symlink() or not (path.is_file() or path.is_dir()):
+ raise RuntimeError(f"Expected a regular Codex package entry: {path}")
+
package_version = normalize_codex_version(codex_version)
_copy_package_tree(python_runtime_root(), staging_dir)
@@ -202,7 +211,12 @@ def stage_python_runtime_package(
pyproject_text = _rewrite_runtime_platform_tag(pyproject_text, platform_tag)
pyproject_path.write_text(pyproject_text)
- _extract_codex_package_archive(package_archive, staged_runtime_package_root(staging_dir))
+ runtime_package_root = staged_runtime_package_root(staging_dir)
+ if package_source.is_dir():
+ shutil.copytree(package_source, runtime_package_root, dirs_exist_ok=True)
+ _validate_codex_package_layout(runtime_package_root, package_source)
+ else:
+ _extract_codex_package_archive(package_source, runtime_package_root)
return staging_dir
@@ -220,7 +234,7 @@ def _extract_codex_package_archive(package_archive: Path, runtime_package_root:
_validate_codex_package_layout(runtime_package_root, package_archive)
-def _validate_codex_package_layout(package_dir: Path, package_archive: Path) -> None:
+def _validate_codex_package_layout(package_dir: Path, package_source: Path) -> None:
missing_entries = []
if not (package_dir / CODEX_PACKAGE_METADATA).is_file():
missing_entries.append(CODEX_PACKAGE_METADATA)
@@ -235,7 +249,7 @@ def _validate_codex_package_layout(package_dir: Path, package_archive: Path) ->
missing_entries.append(str(Path("bin") / runtime_code_mode_host_name()))
if missing_entries:
missing = ", ".join(missing_entries)
- raise RuntimeError(f"Missing Codex package layout entries in {package_archive}: {missing}")
+ raise RuntimeError(f"Missing Codex package layout entries in {package_source}: {missing}")
def _flatten_string_enum_one_of(definition: dict[str, Any]) -> bool:
@@ -1405,9 +1419,9 @@ def build_parser() -> argparse.ArgumentParser:
help="Output directory for the staged runtime package",
)
stage_runtime_parser.add_argument(
- "package_archive",
+ "package_source",
type=Path,
- help="Path to a Codex package .tar.gz archive for this platform.",
+ help="Path to a Codex package directory or .tar.gz archive for this platform.",
)
stage_runtime_parser.add_argument(
"--codex-version",
@@ -1462,7 +1476,7 @@ def run_command(args: argparse.Namespace, ops: CliOps) -> None:
ops.stage_python_runtime_package(
args.staging_dir,
normalize_codex_version(args.codex_version),
- args.package_archive.resolve(),
+ args.package_source.resolve(),
args.platform_tag,
)
diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py
index c89f75d1a1..d9c6617ed7 100644
--- a/sdk/python/tests/test_artifact_workflow_and_binaries.py
+++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py
@@ -864,7 +864,10 @@ def test_stage_runtime_release_can_pin_wheel_platform_tag(tmp_path: Path) -> Non
assert 'platform-tag = "manylinux_2_17_x86_64"' in pyproject
-def test_stage_runtime_release_rejects_incomplete_package_layout(tmp_path: Path) -> None:
[email protected]("source_name", ["codex-package.tar.gz", "codex-package"])
+def test_stage_runtime_release_rejects_incomplete_package_layout(
+ tmp_path: Path, source_name: str
+) -> None:
script = _load_update_script_module()
package_dir = tmp_path / "codex-package"
(package_dir / "bin").mkdir(parents=True)
@@ -872,7 +875,69 @@ def test_stage_runtime_release_rejects_incomplete_package_layout(tmp_path: Path)
_write_package_archive(package_dir, package_archive)
with pytest.raises(RuntimeError, match="Missing Codex package layout entries"):
- script.stage_python_runtime_package(tmp_path / "runtime-stage", "1.2.3", package_archive)
+ script.stage_python_runtime_package(
+ tmp_path / "runtime-stage", "1.2.3", tmp_path / source_name
+ )
+
+
+def test_stage_runtime_directory_matches_archive(tmp_path: Path) -> None:
+ script = _load_update_script_module()
+ package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
+ (package_dir / "bin" / script.runtime_binary_name()).chmod(0o755)
+ package_archive = tmp_path / "codex-package.tar.gz"
+ _write_package_archive(package_dir, package_archive)
+
+ staged_trees = []
+ for name, source in [("archive", package_archive), ("directory", package_dir)]:
+ staged = script.stage_python_runtime_package(
+ tmp_path / name, "1.2.3", source, platform_tag="win_amd64"
+ )
+ staged_trees.append(
+ {
+ path.relative_to(staged): (path.read_bytes(), path.stat().st_mode & 0o777)
+ for path in staged.rglob("*")
+ if path.is_file()
+ }
+ )
+
+ assert staged_trees[0] == staged_trees[1]
+ (script.staged_runtime_package_root(tmp_path / "directory") / "codex-package.json").unlink()
+ assert (package_dir / "codex-package.json").is_file()
+
+
[email protected]("entry_kind", ["file-link", "directory-link", "fifo"])
+def test_stage_runtime_directory_rejects_non_regular_entries(
+ tmp_path: Path, entry_kind: str
+) -> None:
+ script = _load_update_script_module()
+ package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
+ entry = package_dir / "codex-resources" / "invalid"
+ if entry_kind == "fifo":
+ if not hasattr(os, "mkfifo"):
+ pytest.skip("FIFOs are not available on this platform")
+ os.mkfifo(entry)
+ else:
+ target = package_dir / ("codex-package.json" if entry_kind == "file-link" else "bin")
+ try:
+ entry.symlink_to(target, target_is_directory=target.is_dir())
+ except OSError:
+ pytest.skip("Symlinks are not available on this platform")
+
+ with pytest.raises(RuntimeError, match="Expected a regular Codex package entry"):
+ script.stage_python_runtime_package(tmp_path / "runtime-stage", "1.2.3", package_dir)
+
+
[email protected]("staging_path", ["codex-package", "codex-package/stage", "."])
+def test_stage_runtime_directory_rejects_overlapping_staging(
+ tmp_path: Path, staging_path: str
+) -> None:
+ script = _load_update_script_module()
+ package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
+
+ with pytest.raises(RuntimeError, match="directories must not overlap"):
+ script.stage_python_runtime_package(tmp_path / staging_path, "1.2.3", package_dir)
+
+ assert (package_dir / "codex-package.json").is_file()
def test_runtime_package_layout_is_included_by_wheel_config(
@@ -1111,15 +1176,18 @@ def test_sdk_beta_can_use_a_supported_runtime(
)
-def test_stage_runtime_stages_package_without_type_generation(tmp_path: Path) -> None:
[email protected]("source_name", ["codex-package.tar.gz", "codex-package"])
+def test_stage_runtime_stages_package_without_type_generation(
+ tmp_path: Path, source_name: str
+) -> None:
script = _load_update_script_module()
- package_archive = _write_fake_codex_package_archive(tmp_path, script)
+ _write_fake_codex_package_archive(tmp_path, script)
calls: list[str] = []
args = script.parse_args(
[
"stage-runtime",
str(tmp_path / "runtime-stage"),
- str(package_archive),
+ str(tmp_path / source_name),
"--codex-version",
"rust-v0.116.0-alpha.1",
"--platform-tag",
@@ -1152,7 +1220,7 @@ def test_stage_runtime_stages_package_without_type_generation(tmp_path: Path) ->
script.run_command(args, ops)
- assert calls == ["stage_runtime:0.116.0a1:manylinux_2_17_x86_64:codex-package.tar.gz"]
+ assert calls == [f"stage_runtime:0.116.0a1:manylinux_2_17_x86_64:{source_name}"]
def test_default_runtime_is_resolved_from_installed_runtime_package(