OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix: enforce UTF-8 at Windows process boundaries (#501)
faizan-oai5cee678e908d
diff --git a/sdk/typescript/_bundled_plugin/.mcp.json b/sdk/typescript/_bundled_plugin/.mcp.jsonindex d5172719c..dfea7dda8 100644--- a/sdk/typescript/_bundled_plugin/.mcp.json+++ b/sdk/typescript/_bundled_plugin/.mcp.json@@ -32,6 +32,7 @@"AWS_CONTAINER_AUTHORIZATION_TOKEN","AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE","PYTHON",+ "PYTHONUTF8","CODEX_SECURITY_KNOWLEDGE_BASE","CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH","CODEX_SECURITY_SCAN_ROOT",diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.pyindex 0f8e6f87d..8c4dd3722 100644--- a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py+++ b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py@@ -639,20 +639,19 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,],check=True,capture_output=True,- text=True,)- fields = result.stdout.split("\0")+ fields = result.stdout.split(b"\0")if fields and not fields[-1]:fields.pop()changed: list[tuple[Path, str]] = []index = 0while index < len(fields):- status = fields[index][0]+ status = chr(fields[index][0])index += 1if status in {"C", "R"}:index += 1- path = repo / fields[index]+ path = repo / os.fsdecode(fields[index])index += 1changed.append((path, status))return changed@@ -667,14 +666,13 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tupleuntracked = subprocess.run(["git", "-C", str(repo), "ls-files", "--others", "--exclude-standard", "-z"],capture_output=True,- text=True,check=True,)combined = dict(staged)combined.update(unstaged)combined.update(- (repo / relative, "A")- for relative in untracked.stdout.split("\0")+ (repo / os.fsdecode(relative), "A")+ for relative in untracked.stdout.split(b"\0")if relative)return sorted(combined.items())diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.pyindex 746c75a82..bd4079467 100644--- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py+++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py@@ -104,6 +104,7 @@ from workbench_target import (copy_git_worktree_files,directory_content_digest,directory_snapshot_regular_file_count,+ git_bytes,git_command,git_output,git_revision,@@ -315,11 +316,11 @@ def require_diff_target(}if kind == "commit":head = resolve_git_commit(target, head_revision or "", "Commit")- commit = git_output(target, "cat-file", "-p", head)+ commit = git_bytes(target, "cat-file", "-p", head)if commit is None:raise SystemExit(f"Commit is not available in the local checkout: {head}")parent_line = next(- (line for line in commit.splitlines() if line.startswith("parent ")),+ (line for line in commit.splitlines() if line.startswith(b"parent ")),None,)if parent_line is None:@@ -327,7 +328,7 @@ def require_diff_target(else:parent = resolve_git_commit(target,- parent_line.removeprefix("parent ").strip(),+ parent_line.removeprefix(b"parent ").decode("ascii").strip(),"Commit parent",)if base_revision and base_revision != parent:@@ -2888,7 +2889,7 @@ def require_reviewed_patch_applied(applied = git_command(checkout if unversioned else checkout_root,*arguments,- text=True,+ text=False,git_dir=Path(git_dir) if git_dir is not None else None,work_tree=checkout_root if git_dir is not None else None,)diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target.pyindex 01ccb4e6d..3e26d5acd 100644--- a/sdk/typescript/_bundled_plugin/scripts/workbench_target.py+++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target.py@@ -25,8 +25,8 @@ def git_output(git_dir: Path | None = None,work_tree: Path | None = None,) -> str | None:- completed = git_command(target, *args, text=True, git_dir=git_dir, work_tree=work_tree)- output = completed.stdout.strip()+ completed = git_command(target, *args, text=False, git_dir=git_dir, work_tree=work_tree)+ output = os.fsdecode(completed.stdout).strip()return output if completed.returncode == 0 and output else None@@ -154,6 +154,8 @@ def git_command(capture_output=True,env=environment,text=text,+ encoding="utf-8" if text else None,+ errors="surrogateescape" if text else None,input=input_data,)except FileNotFoundError:@@ -539,7 +541,9 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Patif nested_git_dir is None:raise SystemExit(f"Could not inspect nested Git working tree: {relative}")copy_git_worktree_files(source_path, destination_path, excluded)- (destination_path / ".git").write_text(f"gitdir: {nested_git_dir}\n")+ (destination_path / ".git").write_text(+ f"gitdir: {nested_git_dir}\n", encoding="utf-8"+ )else:raise SystemExit(f"Unsupported Git working-tree file type: {relative}")copied_target = destination if pathspec == "." else destination / pathspecdiff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex 3a0acfd2b..d8a2e9d0f 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -107,6 +107,7 @@ import {codexSecurityStateDirectory,expandHome,prepareCodexSecurityCredentialHome,+ pythonUtf8Environment,resolveCodexCommand,resolvePluginPython,runWorkbench,@@ -1069,6 +1070,8 @@ const DEFAULT_DEPENDENCIES: CliDependencies = {python,["-I",+ "-X",+ "utf8",join(plugin, "scripts", "finalize_scan_contract.py"),"--scan-dir",arguments_.scanDir,@@ -1342,24 +1345,26 @@ async function writeCliOutput(export function exportEnvironment(environment: NodeJS.ProcessEnv = process.env,): NodeJS.ProcessEnv {- return Object.fromEntries(- [- "PATH",- "Path",- "PATHEXT",- "SystemRoot",- "SYSTEMROOT",- "WINDIR",- "TMP",- "TEMP",- "TMPDIR",- "PYTHON",- "LANG",- "LC_ALL",- "LC_CTYPE",- ]- .filter((key) => environment[key] !== undefined)- .map((key) => [key, environment[key]]),+ return pythonUtf8Environment(+ Object.fromEntries(+ [+ "PATH",+ "Path",+ "PATHEXT",+ "SystemRoot",+ "SYSTEMROOT",+ "WINDIR",+ "TMP",+ "TEMP",+ "TMPDIR",+ "PYTHON",+ "LANG",+ "LC_ALL",+ "LC_CTYPE",+ ]+ .filter((key) => environment[key] !== undefined)+ .map((key) => [key, environment[key]]),+ ),);}diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.tsindex 3569c11b4..cc5c97b02 100644--- a/sdk/typescript/src/runtime.ts+++ b/sdk/typescript/src/runtime.ts@@ -1386,11 +1386,13 @@ export async function runWorkbench({ command: options.python },["-I",+ "-X",+ "utf8","-B",join(options.pluginRoot, "scripts", "workbench_db.py"),...args,],- environment,+ pythonUtf8Environment(environment),input,options.signal,);@@ -2339,12 +2341,23 @@ export function pluginExecutionEnvironment(environment: ProcessEnvironment = process.env,): ProcessEnvironment {return {- ...environment,+ ...pythonUtf8Environment(environment),PYTHON: python,CODEX_CLI_PATH: resolveCodexCommand(environment).command,};}+export function pythonUtf8Environment(+ environment: ProcessEnvironment,+): ProcessEnvironment {+ const normalized = { ...environment };+ for (const name of Object.keys(normalized)) {+ if (name.toUpperCase() === "PYTHONUTF8") delete normalized[name];+ }+ normalized["PYTHONUTF8"] = "1";+ return normalized;+}+export async function cleanupSdkDirectory(path: string): Promise<void> {await rm(path, { recursive: true, force: true });}diff --git a/sdk/typescript/tests-ts/cli-export.test.ts b/sdk/typescript/tests-ts/cli-export.test.tsindex 78dec14f3..e8e58c8a4 100644--- a/sdk/typescript/tests-ts/cli-export.test.ts+++ b/sdk/typescript/tests-ts/cli-export.test.ts@@ -53,6 +53,7 @@ describe("CLI", () => {).toEqual({Path: "C:\\Python;C:\\Windows\\System32",PYTHON: "/managed/python",+ PYTHONUTF8: "1",TMPDIR: "/tmp",});});diff --git a/sdk/typescript/tests-ts/diff-rank-input.test.ts b/sdk/typescript/tests-ts/diff-rank-input.test.tsindex d4ec49f0c..19e120300 100644--- a/sdk/typescript/tests-ts/diff-rank-input.test.ts+++ b/sdk/typescript/tests-ts/diff-rank-input.test.ts@@ -9,12 +9,21 @@ import {writeFileSync,} from "node:fs";import { tmpdir } from "node:os";-import { join } from "node:path";+import { basename, join } from "node:path";import { afterEach, expect, test } from "bun:test";import { PLUGIN_ROOT } from "./plugin-root.js";const temporaryRoots: string[] = [];+function pythonExecutable(): string | null {+ return (+ process.env["PYTHON"] ??+ Bun.which("python3") ??+ Bun.which("python") ??+ Bun.which("py")+ );+}+afterEach(() => {for (const root of temporaryRoots.splice(0)) {rmSync(root, { recursive: true, force: true });@@ -80,7 +89,7 @@ test("diff previews stay inside the selected repository", () => {rmSync(nested, { recursive: true });symlinkSync(externalFixture, nested, "junction");- const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py");+ const python = pythonExecutable();expect(python).not.toBeNull();const output = join(root, "rank-input.jsonl");const result = spawnSync(@@ -121,3 +130,103 @@ test("diff previews stay inside the selected repository", () => {"",);});++test("preserves Unicode Git paths and legacy-encoded commit metadata", () => {+ const root = realpathSync(+ mkdtempSync(join(tmpdir(), "codex-security-diff-rank-unicode-")),+ );+ temporaryRoots.push(root);+ const repository = join(root, "repository-漢字");+ const source = join(repository, "src", "変更.py");+ mkdirSync(join(repository, "src"), { recursive: true });+ git(repository, "init", "-q");+ writeFileSync(source, "value = 1\n");+ git(repository, "add", ".");+ git(repository, "commit", "-qm", "base");+ const base = git(repository, "rev-parse", "HEAD");+ writeFileSync(source, "value = 2\n");+ git(repository, "add", ".");+ git(repository, "commit", "-qm", "変更");+ const head = git(repository, "rev-parse", "HEAD");+ const legacyMessage = join(root, "legacy-message");+ writeFileSync(legacyMessage, Buffer.from("café\n", "latin1"));+ git(+ repository,+ "-c",+ "i18n.commitEncoding=ISO-8859-1",+ "commit",+ "--allow-empty",+ "-q",+ "-F",+ legacyMessage,+ );+ const legacyHead = git(repository, "rev-parse", "HEAD");++ const python = pythonExecutable();+ expect(python).not.toBeNull();+ const output = join(root, "rank-input.jsonl");+ const rank = spawnSync(+ python!,+ [+ "-I",+ "-B",+ join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),+ "make-diff-rank-input",+ "--repo",+ repository,+ "--base",+ base,+ "--head",+ head,+ "--out",+ output,+ ],+ { encoding: "utf8" },+ );+ const probeSource = [+ "import json, pathlib, sys",+ "sys.path.insert(0, sys.argv[1])",+ "import workbench_target as target",+ "import workbench_db as db",+ "repo = pathlib.Path(sys.argv[2])",+ "root, pathspec = target.git_worktree_context(repo)",+ "metadata = target.git_target_metadata(repo)",+ "diff = db.require_diff_target(repo, 'commit', None, sys.argv[3], None)",+ "print(json.dumps({'root': str(root), 'pathspec': pathspec, 'subject': metadata['commitSubject'], 'diff': diff}))",+ ].join("\n");+ const probe = spawnSync(+ python!,+ [+ "-I",+ "-B",+ "-c",+ probeSource,+ join(PLUGIN_ROOT, "scripts"),+ repository,+ legacyHead,+ ],+ { encoding: "utf8" },+ );++ expect(rank.status, `${rank.stderr}\n${String(rank.error ?? "")}`).toBe(0);+ expect(+ readFileSync(output, "utf8")+ .trimEnd()+ .split("\n")+ .map((row) => JSON.parse(row) as { path: string; preview: string })+ .map(({ path, preview }) => ({ path, preview })),+ ).toEqual([{ path: "src/変更.py", preview: "value = 2" }]);+ expect(probe.status, probe.stderr).toBe(0);+ const target = JSON.parse(probe.stdout) as {+ root: string;+ pathspec: string;+ subject: string;+ diff: { kind: string; baseRevision: string; headRevision: string };+ };+ expect(basename(target.root)).toBe("repository-漢字");+ expect(target).toMatchObject({+ pathspec: ".",+ subject: "café",+ diff: { kind: "commit", baseRevision: head, headRevision: legacyHead },+ });+});diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.tsindex d27a21162..7dbf80c8f 100644--- a/sdk/typescript/tests-ts/runtime.test.ts+++ b/sdk/typescript/tests-ts/runtime.test.ts@@ -2028,6 +2028,7 @@ describe("plugin runtime preparation", () => {expect(workerEnvironment["CODEX_CLI_PATH"]).toBe(resolveCodexCommand().command,);+ expect(workerEnvironment["PYTHONUTF8"]).toBe("1");const globalCodex = spawnSync("codex", ["--version"], {encoding: "utf8",env: workerEnvironment,@@ -2059,6 +2060,7 @@ describe("plugin runtime preparation", () => {CODEX_CLI_PATH: configured,PATH: "",PYTHON: "/managed/python",+ PYTHONUTF8: "1",});expect(pluginExecutionEnvironment("/managed/python", {@@ -3928,7 +3930,7 @@ describe("runtime directories and plugin Python boundary", () => {"assert os.environ.get('OPENROUTER_API_KEY') is None","assert os.environ.get('FIREWORKS_API_KEY') is None","payload = sys.stdin.read()",- "print(json.dumps({'ok': True, 'inputLength': len(payload), 'details': 'x' * (5 * 1024 * 1024)}))",+ "print(json.dumps({'ok': True, 'label': '出力', 'inputLength': len(payload), 'details': 'x' * (5 * 1024 * 1024)}, ensure_ascii=False))",].join("\n"),);const python = Bun.which("python3") ?? Bun.which("python");@@ -3949,6 +3951,7 @@ describe("runtime directories and plugin Python boundary", () => {"x".repeat(64 * 1024),);expect(result["ok"]).toBe(true);+ expect(result["label"]).toBe("出力");expect(result["inputLength"]).toBe(64 * 1024);expect(result["details"]).toHaveLength(5 * 1024 * 1024);});@@ -4892,6 +4895,48 @@ describe("runtime directories and plugin Python boundary", () => {).toBe(await realpath(interpreter!));});+ test.skipIf(process.platform !== "win32")(+ "runs plugin helpers with UTF-8 standard streams",+ async () => {+ const root = await temporaryDirectory("codex-security-python-utf8-");+ const repository = join(root, "repository");+ const output = join(root, "出力.jsonl");+ await mkdir(repository);+ await writeFile(join(repository, "source.py"), "value = 1\n");+ const python = Bun.which("python3") ?? Bun.which("python");+ expect(python).not.toBeNull();++ const result = spawnSync(+ python!,+ [+ "-B",+ join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),+ "make-repo-rank-input",+ "--repo",+ repository,+ "--out",+ output,+ ],+ {+ encoding: "utf8",+ env: pluginExecutionEnvironment(python!, {+ ...process.env,+ pythonutf8: "0",+ }),+ },+ );++ expect(result.status, result.stderr).toBe(0);+ expect(result.stdout).toContain("出力.jsonl");+ expect(+ (await readFile(output, "utf8"))+ .trimEnd()+ .split("\n")+ .map((row) => (JSON.parse(row) as { path: string }).path),+ ).toEqual(["source.py"]);+ },+ );+testPosix("uses configured, inherited, and managed Python", async () => {const root = await temporaryDirectory();const configured = join(root, "configured-python");@@ -4938,6 +4983,7 @@ describe("runtime directories and plugin Python boundary", () => {expect(pluginExecutionEnvironment(managed, { TEST: "1" })).toEqual({TEST: "1",PYTHON: managed,+ PYTHONUTF8: "1",CODEX_CLI_PATH: resolveCodexCommand().command,});await expect(diff --git a/sdk/typescript/tests-ts/workbench-nested-git-utf8.test.ts b/sdk/typescript/tests-ts/workbench-nested-git-utf8.test.tsnew file mode 100644index 000000000..8cab07e2a--- /dev/null+++ b/sdk/typescript/tests-ts/workbench-nested-git-utf8.test.ts@@ -0,0 +1,83 @@+import { spawnSync } from "node:child_process";+import {+ mkdirSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+} from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterEach, expect, test } from "bun:test";+import { PLUGIN_ROOT } from "./plugin-root.js";++const temporaryDirectories: string[] = [];++afterEach(() => {+ for (const directory of temporaryDirectories.splice(0)) {+ rmSync(directory, { recursive: true, force: true });+ }+});++function git(directory: string, ...args: string[]): void {+ const result = spawnSync("git", ["-C", directory, ...args], {+ encoding: "utf8",+ windowsHide: true,+ });+ expect(result.status, result.stderr).toBe(0);+}++function pythonExecutable(): string | null {+ return (+ process.env["PYTHON"] ??+ Bun.which("python3") ??+ Bun.which("python") ??+ Bun.which("py")+ );+}++test("writes nested Git pointers as UTF-8 independently of the locale", () => {+ const root = realpathSync(+ mkdtempSync(join(tmpdir(), "codex-security-nested-git-utf8-")),+ );+ temporaryDirectories.push(root);+ const repository = join(root, "repository");+ const nested = join(repository, "nested-漢字");+ const checkout = join(root, "checkout");+ mkdirSync(nested, { recursive: true });+ git(repository, "init", "-q");+ git(nested, "init", "-q");++ const python = pythonExecutable();+ expect(python).not.toBeNull();+ const probe = [+ "import pathlib, sys",+ "sys.path.insert(0, sys.argv[1])",+ "import workbench_target as target",+ "original_open = pathlib.Path.open",+ "def locale_open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None):",+ " if 'b' not in mode and encoding is None:",+ " encoding = 'cp1252'",+ " return original_open(self, mode, buffering, encoding, errors, newline)",+ "pathlib.Path.open = locale_open",+ "target.copy_git_worktree_files(pathlib.Path(sys.argv[2]), pathlib.Path(sys.argv[3]), ())",+ ].join("\n");+ const result = spawnSync(+ python!,+ [+ "-I",+ "-B",+ "-c",+ probe,+ join(PLUGIN_ROOT, "scripts"),+ repository,+ checkout,+ ],+ { encoding: "utf8", windowsHide: true },+ );++ expect(result.status, result.stderr).toBe(0);+ expect(readFileSync(join(checkout, "nested-漢字", ".git"), "utf8")).toContain(+ "nested-漢字",+ );+});