Lightweight coding agent that runs in your terminal
Rust
Add Bazel targets for on-demand schemas and Python SDK generation (#54096)
Dmitriy Kovalenkobc13ff239cef
diff --git a/MODULE.bazel b/MODULE.bazelindex ab81530b0b..1906a03473 100644--- a/MODULE.bazel+++ b/MODULE.bazel@@ -182,6 +182,23 @@ voice_python.defaults(python_version = "3.11")voice_python.toolchain(python_version = "3.12")use_repo(voice_python, "python_3_12")+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")+pip.parse(+ download_only = True,+ hub_name = "schema_python",+ python_version = "3.12",+ target_platforms = [+ "linux_aarch64",+ "linux_x86_64",+ "osx_aarch64",+ "osx_x86_64",+ "windows_aarch64",+ "windows_x86_64",+ ],+ uv_lock = "//sdk/python:uv.lock",+)+use_repo(pip, "schema_python")+# Use the existing pinned Make build for foreign_cc consumers, including the# native voice pkg-config bootstrap, instead of a PATH-provided Make.register_toolchains("@rules_foreign_cc//toolchains:built_make_toolchain")diff --git a/bazel/rules/schema_bundle.bzl b/bazel/rules/schema_bundle.bzlindex 5f8e2eae01..4a9e559042 100644--- a/bazel/rules/schema_bundle.bzl+++ b/bazel/rules/schema_bundle.bzl@@ -1,4 +1,4 @@-"""Cache schema generation from the filtered public Rust test executable."""+"""Generate API artifacts with the existing Rust and Python exporters."""def _schema_bundle_impl(ctx):outputs = []@@ -27,7 +27,10 @@ def _schema_bundle_impl(ctx):# can normalize the JSON without a separate `bazel run` invocation.zstd = ctx.actions.declare_file(ctx.label.name + ".zstd")ctx.actions.symlink(output = zstd, target_file = ctx.executable.zstd, is_executable = True)- return [DefaultInfo(files = depset(outputs + [zstd]))]+ return [+ DefaultInfo(files = depset(outputs + [zstd])),+ OutputGroupInfo(stable = depset([outputs[0]]), experimental = depset([outputs[1]])),+ ]schema_bundle = rule(implementation = _schema_bundle_impl,@@ -37,3 +40,71 @@ schema_bundle = rule(},doc = "Generates both schema modes as cacheable actions with explicit tool dependencies.",)++def _schema_generator_sources_impl(ctx):+ sources = []+ crate_root = None+ prefix = ctx.label.package + "/"+ for source in ctx.files.srcs:+ if not source.short_path.startswith(prefix):+ fail("Schema generator sources must belong to " + ctx.label.package)+ relative = source.short_path[len(prefix):]+ output = ctx.actions.declare_file(ctx.label.name + "/" + relative)+ ctx.actions.symlink(output = output, target_file = source)+ sources.append(output)+ if relative == "src/lib.rs":+ crate_root = output+ if crate_root == None:+ fail("Schema generator sources must include src/lib.rs")++ # The ignored generation function derives schemas from Rust. These embedded+ # bundles are unused on that path, so do not read the committed artifacts.+ precomputed = [+ ctx.actions.declare_file(ctx.label.name + "/schema/precomputed/app-server-exports-" + mode + ".json.zst")+ for mode in ["stable", "experimental"]+ ]+ for output in precomputed:+ ctx.actions.write(output, "")+ return [+ DefaultInfo(files = depset(sources)),+ OutputGroupInfo(crate_root = depset([crate_root]), precomputed = depset(precomputed)),+ ]++schema_generator_sources = rule(+ implementation = _schema_generator_sources_impl,+ attrs = {"srcs": attr.label_list(allow_files = [".rs"])},+)++def _python_sdk_exports_impl(ctx):+ output = ctx.actions.declare_directory(ctx.label.name)+ manifest = ctx.actions.declare_file(ctx.label.name + ".json")+ prefix = ctx.label.package + "/"+ files = []+ for source in ctx.files.srcs:+ if not source.short_path.startswith(prefix):+ fail("Python SDK generator inputs must belong to " + ctx.label.package)+ files.append([source.path, source.short_path[len(prefix):]])+ ctx.actions.write(manifest, json.encode(files))+ ctx.actions.run(+ executable = ctx.executable._generator,+ arguments = [manifest.path, ctx.file.schema.path + "/json", output.path],+ inputs = ctx.files.srcs + [ctx.file.schema, manifest],+ tools = [ctx.attr._generator[DefaultInfo].files_to_run],+ outputs = [output],+ env = {"PYTHONDONTWRITEBYTECODE": "1", "PYTHONUTF8": "1"},+ mnemonic = "PythonSdkExports",+ )+ return [DefaultInfo(files = depset([output]))]++python_sdk_exports = rule(+ implementation = _python_sdk_exports_impl,+ attrs = {+ "srcs": attr.label_list(allow_files = True),+ "schema": attr.label(mandatory = True, allow_single_file = True),+ "_generator": attr.label(+ default = "//bazel/schema/tools:python-codegen",+ executable = True,+ cfg = "exec",+ ),+ },+)diff --git a/bazel/schema/BUILD.bazel b/bazel/schema/BUILD.bazelindex 401d30b00f..c7dbbc93bf 100644--- a/bazel/schema/BUILD.bazel+++ b/bazel/schema/BUILD.bazel@@ -6,3 +6,18 @@ schema_bundle(generator = "//codex-rs/app-server-protocol:schema-generator",tags = ["manual"],)++schema_bundle(+ name = "app-server",+ generator = "//codex-rs/app-server-protocol:schema-generator-on-demand",+ tags = ["manual"],+ visibility = ["//visibility:public"],+)++filegroup(+ name = "stable",+ srcs = [":app-server"],+ output_group = "stable",+ tags = ["manual"],+ visibility = ["//visibility:public"],+)diff --git a/bazel/schema/tools/BUILD.bazel b/bazel/schema/tools/BUILD.bazelnew file mode 100644index 0000000000..b2ffe5478d--- /dev/null+++ b/bazel/schema/tools/BUILD.bazel@@ -0,0 +1,16 @@+load("@rules_python//python:defs.bzl", "py_binary")+load("@schema_python//:requirements.bzl", "requirement")++package(default_visibility = ["//visibility:public"])++py_binary(+ name = "python-codegen",+ srcs = ["python_codegen.py"],+ main = "python_codegen.py",+ python_version = "3.12",+ tags = ["manual"],+ deps = [+ requirement("datamodel-code-generator"),+ requirement("packaging"),+ ],+)diff --git a/bazel/schema/tools/python_codegen.py b/bazel/schema/tools/python_codegen.pynew file mode 100644index 0000000000..ff3ad00fc3--- /dev/null+++ b/bazel/schema/tools/python_codegen.py@@ -0,0 +1,51 @@+"""Generate SDK build outputs in scratch space without formatting."""++import json+import os+import shutil+import subprocess+import sys+import tempfile+from pathlib import Path+++def main() -> None:+ files = json.loads(Path(sys.argv[1]).read_text())+ schema = Path(sys.argv[2]).resolve()+ output = Path(sys.argv[3]).resolve()+ with tempfile.TemporaryDirectory() as temporary:+ sdk = Path(temporary) / "sdk/python"+ for source, relative in files:+ destination = sdk / relative+ destination.parent.mkdir(parents=True, exist_ok=True)+ shutil.copyfile(source, destination)++ environment = {+ **os.environ,+ "PYTHONPATH": os.pathsep.join(sys.path),+ }+ subprocess.run(+ [+ sys.executable,+ str(sdk / "scripts/update_sdk_artifacts.py"),+ "generate-types",+ "--no-format",+ "--schema-dir",+ str(schema),+ ],+ cwd=sdk,+ env=environment,+ check=True,+ )+ for relative in (+ "src/openai_codex/api.py",+ "src/openai_codex/generated/v2_all.py",+ "src/openai_codex/generated/notification_registry.py",+ ):+ destination = output / relative+ destination.parent.mkdir(parents=True, exist_ok=True)+ shutil.copyfile(sdk / relative, destination)+++if __name__ == "__main__":+ main()diff --git a/codex-rs/app-server-protocol/BUILD.bazel b/codex-rs/app-server-protocol/BUILD.bazelindex 4ecf754479..32ae594c77 100644--- a/codex-rs/app-server-protocol/BUILD.bazel+++ b/codex-rs/app-server-protocol/BUILD.bazel@@ -1,4 +1,7 @@-load("//:defs.bzl", "codex_rust_crate")+load("@crates//:defs.bzl", "all_crate_deps")+load("@rules_rust//rust:defs.bzl", "rust_binary")+load("//:defs.bzl", "WINDOWS_RUSTC_LINK_FLAGS", "codex_rust_crate")+load("//bazel/rules:schema_bundle.bzl", "schema_generator_sources")codex_rust_crate(name = "app-server-protocol",@@ -17,3 +20,45 @@ alias(tags = ["manual"],visibility = ["//bazel/schema:__pkg__"],)++schema_generator_sources(+ name = "schema-generator-sources",+ srcs = glob(["src/**/*.rs"]),+ tags = ["manual"],+)++filegroup(+ name = "schema-generator-root",+ srcs = [":schema-generator-sources"],+ output_group = "crate_root",+ tags = ["manual"],+)++filegroup(+ name = "schema-generator-placeholders",+ srcs = [":schema-generator-sources"],+ output_group = "precomputed",+ tags = ["manual"],+)++# Compile the crate's test harness, including its unit tests.+# Each schema action runs only write_schema_fixtures_from_env via --exact --ignored.+rust_binary(+ name = "schema-generator-on-demand",+ srcs = [":schema-generator-sources"],+ compile_data = [":schema-generator-placeholders"],+ crate_name = "codex_app_server_protocol",+ crate_root = ":schema-generator-root",+ edition = "2024",+ rustc_env = {+ "BAZEL_PACKAGE": "codex-rs/app-server-protocol",+ "CARGO_PKG_NAME": "codex-app-server-protocol",+ },+ rustc_flags = ["--test"] + WINDOWS_RUSTC_LINK_FLAGS,+ tags = ["manual"],+ visibility = ["//bazel/schema:__pkg__"],+ deps = all_crate_deps(+ normal = True,+ normal_dev = True,+ ),+)diff --git a/sdk/python/BUILD.bazel b/sdk/python/BUILD.bazelnew file mode 100644index 0000000000..002e8094db--- /dev/null+++ b/sdk/python/BUILD.bazel@@ -0,0 +1,21 @@+load("//bazel/rules:schema_bundle.bzl", "python_sdk_exports")++exports_files(["uv.lock"])++python_sdk_exports(+ name = "generated",+ srcs = [+ "pyproject.toml",+ "release_version.py",+ "scripts/update_sdk_artifacts.py",+ ] + glob(+ ["src/**/*.py"],+ exclude = [+ "src/openai_codex/generated/notification_registry.py",+ "src/openai_codex/generated/v2_all.py",+ ],+ ),+ schema = "//bazel/schema:stable",+ tags = ["manual"],+ visibility = ["//visibility:public"],+)diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.pyindex 9451aebc20..d6663c65ab 100755--- a/sdk/python/scripts/update_sdk_artifacts.py+++ b/sdk/python/scripts/update_sdk_artifacts.py@@ -14,6 +14,7 @@ import tempfileimport typesimport typingfrom dataclasses import dataclass+from functools import partialfrom pathlib import Pathfrom typing import Any, Callable, Sequence, get_args, get_origin@@ -545,7 +546,7 @@ def _normalized_schema_bundle_text(schema_dir: Path) -> str:return json.dumps(schema, indent=2, sort_keys=True) + "\n"-def generate_v2_all(schema_dir: Path) -> None:+def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None:"""Regenerate the Pydantic v2 protocol model module from app-server schemas."""out_path = sdk_root() / "src" / "openai_codex" / "generated" / "v2_all.py"out_dir = out_path.parent@@ -556,6 +557,9 @@ def generate_v2_all(schema_dir: Path) -> None:with tempfile.TemporaryDirectory() as td:normalized_bundle = Path(td) / schema_bundle_path(schema_dir).namenormalized_bundle.write_text(_normalized_schema_bundle_text(schema_dir))+ if not format_output:+ # The pinned generator accepts an empty formatter list through TOML.+ (Path(td) / "pyproject.toml").write_text("[tool.datamodel-codegen]\nformatters = []\n")run_python_module("datamodel_code_generator",[@@ -582,18 +586,13 @@ def generate_v2_all(schema_dir: Path) -> None:"--use-annotated","--use-union-operator","--disable-timestamp",- # Keep the generated file formatted deterministically so the- # checked-in artifact only changes when the schema does.- "--formatters",- "ruff-format",- ],- cwd=sdk_root(),+ ]+ + (["--formatters", "ruff-format"] if format_output else []),+ cwd=sdk_root() if format_output else Path(td),)_preserve_inline_image_class_names(out_path)_require_nullable_field(out_path, "ChatgptAccount", r"email: str \| None")- _require_nullable_field(- out_path, "McpResourceReadTarget", r"link_id: Annotated\[\n(?: .*\n)+ \]"- )+ _require_nullable_field(out_path, "McpResourceReadTarget", r"link_id: Annotated\[[\s\S]*?\]")_preserve_open_enum(out_path, "CodexErrorInfoValue", allow_empty=True)_preserve_reasoning_effort_enum(out_path)_preserve_thread_source_enum(out_path)@@ -1351,7 +1350,7 @@ def _render_thread_block(turn_fields: list[PublicFieldSpec], *, is_async: bool =return "\n".join(lines)-def generate_public_api_flat_methods() -> None:+def generate_public_api_flat_methods(*, format_output: bool = True) -> None:"""Regenerate the public convenience methods from generated protocol models."""src_dir = sdk_root() / "src"public_api_path = src_dir / "openai_codex" / "api.py"@@ -1400,15 +1399,16 @@ def generate_public_api_flat_methods() -> None:_render_thread_block(turn_start_fields, is_async=True),)public_api_path.write_text(source)- run_python_module("ruff", ["format", str(public_api_path)], cwd=sdk_root())+ if format_output:+ run_python_module("ruff", ["format", str(public_api_path)], cwd=sdk_root())-def generate_types_from_schema_dir(schema_dir: Path) -> None:+def generate_types_from_schema_dir(schema_dir: Path, *, format_output: bool = True) -> None:"""Regenerate every SDK artifact derived from an existing schema directory."""# v2_all is the authoritative generated surface.- generate_v2_all(schema_dir)+ generate_v2_all(schema_dir, format_output=format_output)generate_notification_registry(schema_dir)- generate_public_api_flat_methods()+ generate_public_api_flat_methods(format_output=format_output)def build_parser() -> argparse.ArgumentParser:@@ -1423,6 +1423,12 @@ def build_parser() -> argparse.ArgumentParser:type=Path,help="App-server JSON schema directory (defaults to tool.codex.codegen.schema-dir)",)+ generate_types_parser.add_argument(+ "--no-format",+ dest="format_output",+ action="store_false",+ help="Generate build outputs without running formatters",+ )stage_sdk_parser = subparsers.add_parser("stage-sdk",@@ -1483,9 +1489,13 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:return build_parser().parse_args(list(argv) if argv is not None else None)-def default_cli_ops() -> CliOps:+def default_cli_ops(*, format_output: bool = True) -> CliOps:return CliOps(- generate_types=generate_types_from_schema_dir,+ generate_types=(+ generate_types_from_schema_dir+ if format_output+ else partial(generate_types_from_schema_dir, format_output=False)+ ),stage_python_sdk_package=stage_python_sdk_package,stage_python_runtime_package=stage_python_runtime_package,)@@ -1520,7 +1530,7 @@ def run_command(args: argparse.Namespace, ops: CliOps) -> None:def main(argv: Sequence[str] | None = None, ops: CliOps | None = None) -> None:args = parse_args(argv)- run_command(args, ops or default_cli_ops())+ run_command(args, ops or default_cli_ops(format_output=getattr(args, "format_output", True)))print("Done.")diff --git a/sdk/python/tests/test_contract_generation.py b/sdk/python/tests/test_contract_generation.pyindex 2b610c46bd..11acc14205 100644--- a/sdk/python/tests/test_contract_generation.py+++ b/sdk/python/tests/test_contract_generation.py@@ -1,7 +1,9 @@from __future__ import annotations+import astimport osimport runpy+import shutilimport subprocessimport sysfrom pathlib import Path@@ -36,23 +38,46 @@ def _snapshot_targets(root: Path) -> dict[str, dict[str, bytes] | bytes | None]:return {str(rel_path): _snapshot_target(root, rel_path) for rel_path in GENERATED_TARGETS}-def test_generated_files_are_up_to_date():- """Regenerating from repository schemas should leave reviewed artifacts unchanged."""[email protected]("format_output", [True, False])+def test_generated_files_are_up_to_date(tmp_path, format_output):+ """Both generation modes preserve the reviewed API, with optional formatting."""before = _snapshot_targets(ROOT)+ sdk = tmp_path / "sdk" / "python"+ sdk.mkdir(parents=True)+ for directory in ("src", "scripts"):+ shutil.copytree(+ ROOT / directory, sdk / directory, ignore=shutil.ignore_patterns("__pycache__")+ )+ for name in ("pyproject.toml", "release_version.py"):+ shutil.copyfile(ROOT / name, sdk / name)env = os.environ.copy()python_bin = str(Path(sys.executable).parent)env["PATH"] = f"{python_bin}{os.pathsep}{env.get('PATH', '')}"+ command = [+ sys.executable,+ "scripts/update_sdk_artifacts.py",+ "generate-types",+ "--schema-dir",+ str(ROOT.parents[1] / "codex-rs/app-server-protocol/schema/json"),+ ]+ if not format_output:+ command.append("--no-format")subprocess.run(- [sys.executable, "scripts/update_sdk_artifacts.py", "generate-types"],- cwd=ROOT,+ command,+ cwd=sdk,check=True,env=env,)- after = _snapshot_targets(ROOT)- assert before == after, "Generated files drifted after regeneration"+ after = _snapshot_targets(sdk)+ if format_output:+ assert before == after, "Generated files drifted after regeneration"+ else:+ assert [ast.dump(ast.parse((ROOT / path).read_bytes())) for path in GENERATED_TARGETS] == [+ ast.dump(ast.parse((sdk / path).read_bytes())) for path in GENERATED_TARGETS+ ]@pytest.mark.parametrize("mode", ["repository", "scratch", "experimental"])