Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add Bazel targets for on-demand schemas and Python SDK generation (#54096)

Dmitriy Kovalenkobc13ff239cef
diff --git a/MODULE.bazel b/MODULE.bazel
index ab81530b0b..1906a03473 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -182,6 +182,23 @@ voice_python.defaults(python_version = "3.11")
voice_python.toolchain(python_version = "3.12")
use_repo(voice_python, "python_3_12")
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+ download_only = True,
+ hub_name = "schema_python",
+ python_version = "3.12",
+ target_platforms = [
+ "linux_aarch64",
+ "linux_x86_64",
+ "osx_aarch64",
+ "osx_x86_64",
+ "windows_aarch64",
+ "windows_x86_64",
+ ],
+ uv_lock = "//sdk/python:uv.lock",
+)
+use_repo(pip, "schema_python")
+
# Use the existing pinned Make build for foreign_cc consumers, including the
# native voice pkg-config bootstrap, instead of a PATH-provided Make.
register_toolchains("@rules_foreign_cc//toolchains:built_make_toolchain")
diff --git a/bazel/rules/schema_bundle.bzl b/bazel/rules/schema_bundle.bzl
index 5f8e2eae01..4a9e559042 100644
--- a/bazel/rules/schema_bundle.bzl
+++ b/bazel/rules/schema_bundle.bzl
@@ -1,4 +1,4 @@
-"""Cache schema generation from the filtered public Rust test executable."""
+"""Generate API artifacts with the existing Rust and Python exporters."""
def _schema_bundle_impl(ctx):
outputs = []
@@ -27,7 +27,10 @@ def _schema_bundle_impl(ctx):
# can normalize the JSON without a separate `bazel run` invocation.
zstd = ctx.actions.declare_file(ctx.label.name + ".zstd")
ctx.actions.symlink(output = zstd, target_file = ctx.executable.zstd, is_executable = True)
- return [DefaultInfo(files = depset(outputs + [zstd]))]
+ return [
+ DefaultInfo(files = depset(outputs + [zstd])),
+ OutputGroupInfo(stable = depset([outputs[0]]), experimental = depset([outputs[1]])),
+ ]
schema_bundle = rule(
implementation = _schema_bundle_impl,
@@ -37,3 +40,71 @@ schema_bundle = rule(
},
doc = "Generates both schema modes as cacheable actions with explicit tool dependencies.",
)
+
+def _schema_generator_sources_impl(ctx):
+ sources = []
+ crate_root = None
+ prefix = ctx.label.package + "/"
+ for source in ctx.files.srcs:
+ if not source.short_path.startswith(prefix):
+ fail("Schema generator sources must belong to " + ctx.label.package)
+ relative = source.short_path[len(prefix):]
+ output = ctx.actions.declare_file(ctx.label.name + "/" + relative)
+ ctx.actions.symlink(output = output, target_file = source)
+ sources.append(output)
+ if relative == "src/lib.rs":
+ crate_root = output
+ if crate_root == None:
+ fail("Schema generator sources must include src/lib.rs")
+
+ # The ignored generation function derives schemas from Rust. These embedded
+ # bundles are unused on that path, so do not read the committed artifacts.
+ precomputed = [
+ ctx.actions.declare_file(ctx.label.name + "/schema/precomputed/app-server-exports-" + mode + ".json.zst")
+ for mode in ["stable", "experimental"]
+ ]
+ for output in precomputed:
+ ctx.actions.write(output, "")
+ return [
+ DefaultInfo(files = depset(sources)),
+ OutputGroupInfo(crate_root = depset([crate_root]), precomputed = depset(precomputed)),
+ ]
+
+schema_generator_sources = rule(
+ implementation = _schema_generator_sources_impl,
+ attrs = {"srcs": attr.label_list(allow_files = [".rs"])},
+)
+
+def _python_sdk_exports_impl(ctx):
+ output = ctx.actions.declare_directory(ctx.label.name)
+ manifest = ctx.actions.declare_file(ctx.label.name + ".json")
+ prefix = ctx.label.package + "/"
+ files = []
+ for source in ctx.files.srcs:
+ if not source.short_path.startswith(prefix):
+ fail("Python SDK generator inputs must belong to " + ctx.label.package)
+ files.append([source.path, source.short_path[len(prefix):]])
+ ctx.actions.write(manifest, json.encode(files))
+ ctx.actions.run(
+ executable = ctx.executable._generator,
+ arguments = [manifest.path, ctx.file.schema.path + "/json", output.path],
+ inputs = ctx.files.srcs + [ctx.file.schema, manifest],
+ tools = [ctx.attr._generator[DefaultInfo].files_to_run],
+ outputs = [output],
+ env = {"PYTHONDONTWRITEBYTECODE": "1", "PYTHONUTF8": "1"},
+ mnemonic = "PythonSdkExports",
+ )
+ return [DefaultInfo(files = depset([output]))]
+
+python_sdk_exports = rule(
+ implementation = _python_sdk_exports_impl,
+ attrs = {
+ "srcs": attr.label_list(allow_files = True),
+ "schema": attr.label(mandatory = True, allow_single_file = True),
+ "_generator": attr.label(
+ default = "//bazel/schema/tools:python-codegen",
+ executable = True,
+ cfg = "exec",
+ ),
+ },
+)
diff --git a/bazel/schema/BUILD.bazel b/bazel/schema/BUILD.bazel
index 401d30b00f..c7dbbc93bf 100644
--- a/bazel/schema/BUILD.bazel
+++ b/bazel/schema/BUILD.bazel
@@ -6,3 +6,18 @@ schema_bundle(
generator = "//codex-rs/app-server-protocol:schema-generator",
tags = ["manual"],
)
+
+schema_bundle(
+ name = "app-server",
+ generator = "//codex-rs/app-server-protocol:schema-generator-on-demand",
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
+
+filegroup(
+ name = "stable",
+ srcs = [":app-server"],
+ output_group = "stable",
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
diff --git a/bazel/schema/tools/BUILD.bazel b/bazel/schema/tools/BUILD.bazel
new file mode 100644
index 0000000000..b2ffe5478d
--- /dev/null
+++ b/bazel/schema/tools/BUILD.bazel
@@ -0,0 +1,16 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@schema_python//:requirements.bzl", "requirement")
+
+package(default_visibility = ["//visibility:public"])
+
+py_binary(
+ name = "python-codegen",
+ srcs = ["python_codegen.py"],
+ main = "python_codegen.py",
+ python_version = "3.12",
+ tags = ["manual"],
+ deps = [
+ requirement("datamodel-code-generator"),
+ requirement("packaging"),
+ ],
+)
diff --git a/bazel/schema/tools/python_codegen.py b/bazel/schema/tools/python_codegen.py
new file mode 100644
index 0000000000..ff3ad00fc3
--- /dev/null
+++ b/bazel/schema/tools/python_codegen.py
@@ -0,0 +1,51 @@
+"""Generate SDK build outputs in scratch space without formatting."""
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def main() -> None:
+ files = json.loads(Path(sys.argv[1]).read_text())
+ schema = Path(sys.argv[2]).resolve()
+ output = Path(sys.argv[3]).resolve()
+ with tempfile.TemporaryDirectory() as temporary:
+ sdk = Path(temporary) / "sdk/python"
+ for source, relative in files:
+ destination = sdk / relative
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(source, destination)
+
+ environment = {
+ **os.environ,
+ "PYTHONPATH": os.pathsep.join(sys.path),
+ }
+ subprocess.run(
+ [
+ sys.executable,
+ str(sdk / "scripts/update_sdk_artifacts.py"),
+ "generate-types",
+ "--no-format",
+ "--schema-dir",
+ str(schema),
+ ],
+ cwd=sdk,
+ env=environment,
+ check=True,
+ )
+ for relative in (
+ "src/openai_codex/api.py",
+ "src/openai_codex/generated/v2_all.py",
+ "src/openai_codex/generated/notification_registry.py",
+ ):
+ destination = output / relative
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(sdk / relative, destination)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/codex-rs/app-server-protocol/BUILD.bazel b/codex-rs/app-server-protocol/BUILD.bazel
index 4ecf754479..32ae594c77 100644
--- a/codex-rs/app-server-protocol/BUILD.bazel
+++ b/codex-rs/app-server-protocol/BUILD.bazel
@@ -1,4 +1,7 @@
-load("//:defs.bzl", "codex_rust_crate")
+load("@crates//:defs.bzl", "all_crate_deps")
+load("@rules_rust//rust:defs.bzl", "rust_binary")
+load("//:defs.bzl", "WINDOWS_RUSTC_LINK_FLAGS", "codex_rust_crate")
+load("//bazel/rules:schema_bundle.bzl", "schema_generator_sources")
codex_rust_crate(
name = "app-server-protocol",
@@ -17,3 +20,45 @@ alias(
tags = ["manual"],
visibility = ["//bazel/schema:__pkg__"],
)
+
+schema_generator_sources(
+ name = "schema-generator-sources",
+ srcs = glob(["src/**/*.rs"]),
+ tags = ["manual"],
+)
+
+filegroup(
+ name = "schema-generator-root",
+ srcs = [":schema-generator-sources"],
+ output_group = "crate_root",
+ tags = ["manual"],
+)
+
+filegroup(
+ name = "schema-generator-placeholders",
+ srcs = [":schema-generator-sources"],
+ output_group = "precomputed",
+ tags = ["manual"],
+)
+
+# Compile the crate's test harness, including its unit tests.
+# Each schema action runs only write_schema_fixtures_from_env via --exact --ignored.
+rust_binary(
+ name = "schema-generator-on-demand",
+ srcs = [":schema-generator-sources"],
+ compile_data = [":schema-generator-placeholders"],
+ crate_name = "codex_app_server_protocol",
+ crate_root = ":schema-generator-root",
+ edition = "2024",
+ rustc_env = {
+ "BAZEL_PACKAGE": "codex-rs/app-server-protocol",
+ "CARGO_PKG_NAME": "codex-app-server-protocol",
+ },
+ rustc_flags = ["--test"] + WINDOWS_RUSTC_LINK_FLAGS,
+ tags = ["manual"],
+ visibility = ["//bazel/schema:__pkg__"],
+ deps = all_crate_deps(
+ normal = True,
+ normal_dev = True,
+ ),
+)
diff --git a/sdk/python/BUILD.bazel b/sdk/python/BUILD.bazel
new file mode 100644
index 0000000000..002e8094db
--- /dev/null
+++ b/sdk/python/BUILD.bazel
@@ -0,0 +1,21 @@
+load("//bazel/rules:schema_bundle.bzl", "python_sdk_exports")
+
+exports_files(["uv.lock"])
+
+python_sdk_exports(
+ name = "generated",
+ srcs = [
+ "pyproject.toml",
+ "release_version.py",
+ "scripts/update_sdk_artifacts.py",
+ ] + glob(
+ ["src/**/*.py"],
+ exclude = [
+ "src/openai_codex/generated/notification_registry.py",
+ "src/openai_codex/generated/v2_all.py",
+ ],
+ ),
+ schema = "//bazel/schema:stable",
+ tags = ["manual"],
+ visibility = ["//visibility:public"],
+)
diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py
index 9451aebc20..d6663c65ab 100755
--- a/sdk/python/scripts/update_sdk_artifacts.py
+++ b/sdk/python/scripts/update_sdk_artifacts.py
@@ -14,6 +14,7 @@ import tempfile
import types
import typing
from dataclasses import dataclass
+from functools import partial
from pathlib import Path
from typing import Any, Callable, Sequence, get_args, get_origin
@@ -545,7 +546,7 @@ def _normalized_schema_bundle_text(schema_dir: Path) -> str:
return json.dumps(schema, indent=2, sort_keys=True) + "\n"
-def generate_v2_all(schema_dir: Path) -> None:
+def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None:
"""Regenerate the Pydantic v2 protocol model module from app-server schemas."""
out_path = sdk_root() / "src" / "openai_codex" / "generated" / "v2_all.py"
out_dir = out_path.parent
@@ -556,6 +557,9 @@ def generate_v2_all(schema_dir: Path) -> None:
with tempfile.TemporaryDirectory() as td:
normalized_bundle = Path(td) / schema_bundle_path(schema_dir).name
normalized_bundle.write_text(_normalized_schema_bundle_text(schema_dir))
+ if not format_output:
+ # The pinned generator accepts an empty formatter list through TOML.
+ (Path(td) / "pyproject.toml").write_text("[tool.datamodel-codegen]\nformatters = []\n")
run_python_module(
"datamodel_code_generator",
[
@@ -582,18 +586,13 @@ def generate_v2_all(schema_dir: Path) -> None:
"--use-annotated",
"--use-union-operator",
"--disable-timestamp",
- # Keep the generated file formatted deterministically so the
- # checked-in artifact only changes when the schema does.
- "--formatters",
- "ruff-format",
- ],
- cwd=sdk_root(),
+ ]
+ + (["--formatters", "ruff-format"] if format_output else []),
+ cwd=sdk_root() if format_output else Path(td),
)
_preserve_inline_image_class_names(out_path)
_require_nullable_field(out_path, "ChatgptAccount", r"email: str \| None")
- _require_nullable_field(
- out_path, "McpResourceReadTarget", r"link_id: Annotated\[\n(?: .*\n)+ \]"
- )
+ _require_nullable_field(out_path, "McpResourceReadTarget", r"link_id: Annotated\[[\s\S]*?\]")
_preserve_open_enum(out_path, "CodexErrorInfoValue", allow_empty=True)
_preserve_reasoning_effort_enum(out_path)
_preserve_thread_source_enum(out_path)
@@ -1351,7 +1350,7 @@ def _render_thread_block(turn_fields: list[PublicFieldSpec], *, is_async: bool =
return "\n".join(lines)
-def generate_public_api_flat_methods() -> None:
+def generate_public_api_flat_methods(*, format_output: bool = True) -> None:
"""Regenerate the public convenience methods from generated protocol models."""
src_dir = sdk_root() / "src"
public_api_path = src_dir / "openai_codex" / "api.py"
@@ -1400,15 +1399,16 @@ def generate_public_api_flat_methods() -> None:
_render_thread_block(turn_start_fields, is_async=True),
)
public_api_path.write_text(source)
- run_python_module("ruff", ["format", str(public_api_path)], cwd=sdk_root())
+ if format_output:
+ run_python_module("ruff", ["format", str(public_api_path)], cwd=sdk_root())
-def generate_types_from_schema_dir(schema_dir: Path) -> None:
+def generate_types_from_schema_dir(schema_dir: Path, *, format_output: bool = True) -> None:
"""Regenerate every SDK artifact derived from an existing schema directory."""
# v2_all is the authoritative generated surface.
- generate_v2_all(schema_dir)
+ generate_v2_all(schema_dir, format_output=format_output)
generate_notification_registry(schema_dir)
- generate_public_api_flat_methods()
+ generate_public_api_flat_methods(format_output=format_output)
def build_parser() -> argparse.ArgumentParser:
@@ -1423,6 +1423,12 @@ def build_parser() -> argparse.ArgumentParser:
type=Path,
help="App-server JSON schema directory (defaults to tool.codex.codegen.schema-dir)",
)
+ generate_types_parser.add_argument(
+ "--no-format",
+ dest="format_output",
+ action="store_false",
+ help="Generate build outputs without running formatters",
+ )
stage_sdk_parser = subparsers.add_parser(
"stage-sdk",
@@ -1483,9 +1489,13 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
return build_parser().parse_args(list(argv) if argv is not None else None)
-def default_cli_ops() -> CliOps:
+def default_cli_ops(*, format_output: bool = True) -> CliOps:
return CliOps(
- generate_types=generate_types_from_schema_dir,
+ generate_types=(
+ generate_types_from_schema_dir
+ if format_output
+ else partial(generate_types_from_schema_dir, format_output=False)
+ ),
stage_python_sdk_package=stage_python_sdk_package,
stage_python_runtime_package=stage_python_runtime_package,
)
@@ -1520,7 +1530,7 @@ def run_command(args: argparse.Namespace, ops: CliOps) -> None:
def main(argv: Sequence[str] | None = None, ops: CliOps | None = None) -> None:
args = parse_args(argv)
- run_command(args, ops or default_cli_ops())
+ run_command(args, ops or default_cli_ops(format_output=getattr(args, "format_output", True)))
print("Done.")
diff --git a/sdk/python/tests/test_contract_generation.py b/sdk/python/tests/test_contract_generation.py
index 2b610c46bd..11acc14205 100644
--- a/sdk/python/tests/test_contract_generation.py
+++ b/sdk/python/tests/test_contract_generation.py
@@ -1,7 +1,9 @@
from __future__ import annotations
+import ast
import os
import runpy
+import shutil
import subprocess
import sys
from pathlib import Path
@@ -36,23 +38,46 @@ def _snapshot_targets(root: Path) -> dict[str, dict[str, bytes] | bytes | None]:
return {str(rel_path): _snapshot_target(root, rel_path) for rel_path in GENERATED_TARGETS}
-def test_generated_files_are_up_to_date():
- """Regenerating from repository schemas should leave reviewed artifacts unchanged."""
[email protected]("format_output", [True, False])
+def test_generated_files_are_up_to_date(tmp_path, format_output):
+ """Both generation modes preserve the reviewed API, with optional formatting."""
before = _snapshot_targets(ROOT)
+ sdk = tmp_path / "sdk" / "python"
+ sdk.mkdir(parents=True)
+ for directory in ("src", "scripts"):
+ shutil.copytree(
+ ROOT / directory, sdk / directory, ignore=shutil.ignore_patterns("__pycache__")
+ )
+ for name in ("pyproject.toml", "release_version.py"):
+ shutil.copyfile(ROOT / name, sdk / name)
env = os.environ.copy()
python_bin = str(Path(sys.executable).parent)
env["PATH"] = f"{python_bin}{os.pathsep}{env.get('PATH', '')}"
+ command = [
+ sys.executable,
+ "scripts/update_sdk_artifacts.py",
+ "generate-types",
+ "--schema-dir",
+ str(ROOT.parents[1] / "codex-rs/app-server-protocol/schema/json"),
+ ]
+ if not format_output:
+ command.append("--no-format")
subprocess.run(
- [sys.executable, "scripts/update_sdk_artifacts.py", "generate-types"],
- cwd=ROOT,
+ command,
+ cwd=sdk,
check=True,
env=env,
)
- after = _snapshot_targets(ROOT)
- assert before == after, "Generated files drifted after regeneration"
+ after = _snapshot_targets(sdk)
+ if format_output:
+ assert before == after, "Generated files drifted after regeneration"
+ else:
+ assert [ast.dump(ast.parse((ROOT / path).read_bytes())) for path in GENERATED_TARGETS] == [
+ ast.dump(ast.parse((sdk / path).read_bytes())) for path in GENERATED_TARGETS
+ ]
@pytest.mark.parametrize("mode", ["repository", "scratch", "experimental"])