Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: enforce UTF-8 at Windows process boundaries (#501)

faizan-oai5cee678e908d
diff --git a/sdk/typescript/_bundled_plugin/.mcp.json b/sdk/typescript/_bundled_plugin/.mcp.json
index d5172719c..dfea7dda8 100644
--- a/sdk/typescript/_bundled_plugin/.mcp.json
+++ b/sdk/typescript/_bundled_plugin/.mcp.json
@@ -32,6 +32,7 @@
"AWS_CONTAINER_AUTHORIZATION_TOKEN",
"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"PYTHON",
+ "PYTHONUTF8",
"CODEX_SECURITY_KNOWLEDGE_BASE",
"CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH",
"CODEX_SECURITY_SCAN_ROOT",
diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py
index 0f8e6f87d..8c4dd3722 100644
--- a/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py
+++ b/sdk/typescript/_bundled_plugin/scripts/generate_rank_input.py
@@ -639,20 +639,19 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,
],
check=True,
capture_output=True,
- text=True,
)
- fields = result.stdout.split("\0")
+ fields = result.stdout.split(b"\0")
if fields and not fields[-1]:
fields.pop()
changed: list[tuple[Path, str]] = []
index = 0
while index < len(fields):
- status = fields[index][0]
+ status = chr(fields[index][0])
index += 1
if status in {"C", "R"}:
index += 1
- path = repo / fields[index]
+ path = repo / os.fsdecode(fields[index])
index += 1
changed.append((path, status))
return changed
@@ -667,14 +666,13 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
untracked = subprocess.run(
["git", "-C", str(repo), "ls-files", "--others", "--exclude-standard", "-z"],
capture_output=True,
- text=True,
check=True,
)
combined = dict(staged)
combined.update(unstaged)
combined.update(
- (repo / relative, "A")
- for relative in untracked.stdout.split("\0")
+ (repo / os.fsdecode(relative), "A")
+ for relative in untracked.stdout.split(b"\0")
if relative
)
return sorted(combined.items())
diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py
index 746c75a82..bd4079467 100644
--- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py
+++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py
@@ -104,6 +104,7 @@ from workbench_target import (
copy_git_worktree_files,
directory_content_digest,
directory_snapshot_regular_file_count,
+ git_bytes,
git_command,
git_output,
git_revision,
@@ -315,11 +316,11 @@ def require_diff_target(
}
if kind == "commit":
head = resolve_git_commit(target, head_revision or "", "Commit")
- commit = git_output(target, "cat-file", "-p", head)
+ commit = git_bytes(target, "cat-file", "-p", head)
if commit is None:
raise SystemExit(f"Commit is not available in the local checkout: {head}")
parent_line = next(
- (line for line in commit.splitlines() if line.startswith("parent ")),
+ (line for line in commit.splitlines() if line.startswith(b"parent ")),
None,
)
if parent_line is None:
@@ -327,7 +328,7 @@ def require_diff_target(
else:
parent = resolve_git_commit(
target,
- parent_line.removeprefix("parent ").strip(),
+ parent_line.removeprefix(b"parent ").decode("ascii").strip(),
"Commit parent",
)
if base_revision and base_revision != parent:
@@ -2888,7 +2889,7 @@ def require_reviewed_patch_applied(
applied = git_command(
checkout if unversioned else checkout_root,
*arguments,
- text=True,
+ text=False,
git_dir=Path(git_dir) if git_dir is not None else None,
work_tree=checkout_root if git_dir is not None else None,
)
diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target.py
index 01ccb4e6d..3e26d5acd 100644
--- a/sdk/typescript/_bundled_plugin/scripts/workbench_target.py
+++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target.py
@@ -25,8 +25,8 @@ def git_output(
git_dir: Path | None = None,
work_tree: Path | None = None,
) -> str | None:
- completed = git_command(target, *args, text=True, git_dir=git_dir, work_tree=work_tree)
- output = completed.stdout.strip()
+ completed = git_command(target, *args, text=False, git_dir=git_dir, work_tree=work_tree)
+ output = os.fsdecode(completed.stdout).strip()
return output if completed.returncode == 0 and output else None
@@ -154,6 +154,8 @@ def git_command(
capture_output=True,
env=environment,
text=text,
+ encoding="utf-8" if text else None,
+ errors="surrogateescape" if text else None,
input=input_data,
)
except FileNotFoundError:
@@ -539,7 +541,9 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat
if nested_git_dir is None:
raise SystemExit(f"Could not inspect nested Git working tree: {relative}")
copy_git_worktree_files(source_path, destination_path, excluded)
- (destination_path / ".git").write_text(f"gitdir: {nested_git_dir}\n")
+ (destination_path / ".git").write_text(
+ f"gitdir: {nested_git_dir}\n", encoding="utf-8"
+ )
else:
raise SystemExit(f"Unsupported Git working-tree file type: {relative}")
copied_target = destination if pathspec == "." else destination / pathspec
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 3a0acfd2b..d8a2e9d0f 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -107,6 +107,7 @@ import {
codexSecurityStateDirectory,
expandHome,
prepareCodexSecurityCredentialHome,
+ pythonUtf8Environment,
resolveCodexCommand,
resolvePluginPython,
runWorkbench,
@@ -1069,6 +1070,8 @@ const DEFAULT_DEPENDENCIES: CliDependencies = {
python,
[
"-I",
+ "-X",
+ "utf8",
join(plugin, "scripts", "finalize_scan_contract.py"),
"--scan-dir",
arguments_.scanDir,
@@ -1342,24 +1345,26 @@ async function writeCliOutput(
export function exportEnvironment(
environment: NodeJS.ProcessEnv = process.env,
): NodeJS.ProcessEnv {
- return Object.fromEntries(
- [
- "PATH",
- "Path",
- "PATHEXT",
- "SystemRoot",
- "SYSTEMROOT",
- "WINDIR",
- "TMP",
- "TEMP",
- "TMPDIR",
- "PYTHON",
- "LANG",
- "LC_ALL",
- "LC_CTYPE",
- ]
- .filter((key) => environment[key] !== undefined)
- .map((key) => [key, environment[key]]),
+ return pythonUtf8Environment(
+ Object.fromEntries(
+ [
+ "PATH",
+ "Path",
+ "PATHEXT",
+ "SystemRoot",
+ "SYSTEMROOT",
+ "WINDIR",
+ "TMP",
+ "TEMP",
+ "TMPDIR",
+ "PYTHON",
+ "LANG",
+ "LC_ALL",
+ "LC_CTYPE",
+ ]
+ .filter((key) => environment[key] !== undefined)
+ .map((key) => [key, environment[key]]),
+ ),
);
}
diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts
index 3569c11b4..cc5c97b02 100644
--- a/sdk/typescript/src/runtime.ts
+++ b/sdk/typescript/src/runtime.ts
@@ -1386,11 +1386,13 @@ export async function runWorkbench(
{ command: options.python },
[
"-I",
+ "-X",
+ "utf8",
"-B",
join(options.pluginRoot, "scripts", "workbench_db.py"),
...args,
],
- environment,
+ pythonUtf8Environment(environment),
input,
options.signal,
);
@@ -2339,12 +2341,23 @@ export function pluginExecutionEnvironment(
environment: ProcessEnvironment = process.env,
): ProcessEnvironment {
return {
- ...environment,
+ ...pythonUtf8Environment(environment),
PYTHON: python,
CODEX_CLI_PATH: resolveCodexCommand(environment).command,
};
}
+export function pythonUtf8Environment(
+ environment: ProcessEnvironment,
+): ProcessEnvironment {
+ const normalized = { ...environment };
+ for (const name of Object.keys(normalized)) {
+ if (name.toUpperCase() === "PYTHONUTF8") delete normalized[name];
+ }
+ normalized["PYTHONUTF8"] = "1";
+ return normalized;
+}
+
export async function cleanupSdkDirectory(path: string): Promise<void> {
await rm(path, { recursive: true, force: true });
}
diff --git a/sdk/typescript/tests-ts/cli-export.test.ts b/sdk/typescript/tests-ts/cli-export.test.ts
index 78dec14f3..e8e58c8a4 100644
--- a/sdk/typescript/tests-ts/cli-export.test.ts
+++ b/sdk/typescript/tests-ts/cli-export.test.ts
@@ -53,6 +53,7 @@ describe("CLI", () => {
).toEqual({
Path: "C:\\Python;C:\\Windows\\System32",
PYTHON: "/managed/python",
+ PYTHONUTF8: "1",
TMPDIR: "/tmp",
});
});
diff --git a/sdk/typescript/tests-ts/diff-rank-input.test.ts b/sdk/typescript/tests-ts/diff-rank-input.test.ts
index d4ec49f0c..19e120300 100644
--- a/sdk/typescript/tests-ts/diff-rank-input.test.ts
+++ b/sdk/typescript/tests-ts/diff-rank-input.test.ts
@@ -9,12 +9,21 @@ import {
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { basename, join } from "node:path";
import { afterEach, expect, test } from "bun:test";
import { PLUGIN_ROOT } from "./plugin-root.js";
const temporaryRoots: string[] = [];
+function pythonExecutable(): string | null {
+ return (
+ process.env["PYTHON"] ??
+ Bun.which("python3") ??
+ Bun.which("python") ??
+ Bun.which("py")
+ );
+}
+
afterEach(() => {
for (const root of temporaryRoots.splice(0)) {
rmSync(root, { recursive: true, force: true });
@@ -80,7 +89,7 @@ test("diff previews stay inside the selected repository", () => {
rmSync(nested, { recursive: true });
symlinkSync(externalFixture, nested, "junction");
- const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py");
+ const python = pythonExecutable();
expect(python).not.toBeNull();
const output = join(root, "rank-input.jsonl");
const result = spawnSync(
@@ -121,3 +130,103 @@ test("diff previews stay inside the selected repository", () => {
"",
);
});
+
+test("preserves Unicode Git paths and legacy-encoded commit metadata", () => {
+ const root = realpathSync(
+ mkdtempSync(join(tmpdir(), "codex-security-diff-rank-unicode-")),
+ );
+ temporaryRoots.push(root);
+ const repository = join(root, "repository-漢字");
+ const source = join(repository, "src", "変更.py");
+ mkdirSync(join(repository, "src"), { recursive: true });
+ git(repository, "init", "-q");
+ writeFileSync(source, "value = 1\n");
+ git(repository, "add", ".");
+ git(repository, "commit", "-qm", "base");
+ const base = git(repository, "rev-parse", "HEAD");
+ writeFileSync(source, "value = 2\n");
+ git(repository, "add", ".");
+ git(repository, "commit", "-qm", "変更");
+ const head = git(repository, "rev-parse", "HEAD");
+ const legacyMessage = join(root, "legacy-message");
+ writeFileSync(legacyMessage, Buffer.from("café\n", "latin1"));
+ git(
+ repository,
+ "-c",
+ "i18n.commitEncoding=ISO-8859-1",
+ "commit",
+ "--allow-empty",
+ "-q",
+ "-F",
+ legacyMessage,
+ );
+ const legacyHead = git(repository, "rev-parse", "HEAD");
+
+ const python = pythonExecutable();
+ expect(python).not.toBeNull();
+ const output = join(root, "rank-input.jsonl");
+ const rank = spawnSync(
+ python!,
+ [
+ "-I",
+ "-B",
+ join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),
+ "make-diff-rank-input",
+ "--repo",
+ repository,
+ "--base",
+ base,
+ "--head",
+ head,
+ "--out",
+ output,
+ ],
+ { encoding: "utf8" },
+ );
+ const probeSource = [
+ "import json, pathlib, sys",
+ "sys.path.insert(0, sys.argv[1])",
+ "import workbench_target as target",
+ "import workbench_db as db",
+ "repo = pathlib.Path(sys.argv[2])",
+ "root, pathspec = target.git_worktree_context(repo)",
+ "metadata = target.git_target_metadata(repo)",
+ "diff = db.require_diff_target(repo, 'commit', None, sys.argv[3], None)",
+ "print(json.dumps({'root': str(root), 'pathspec': pathspec, 'subject': metadata['commitSubject'], 'diff': diff}))",
+ ].join("\n");
+ const probe = spawnSync(
+ python!,
+ [
+ "-I",
+ "-B",
+ "-c",
+ probeSource,
+ join(PLUGIN_ROOT, "scripts"),
+ repository,
+ legacyHead,
+ ],
+ { encoding: "utf8" },
+ );
+
+ expect(rank.status, `${rank.stderr}\n${String(rank.error ?? "")}`).toBe(0);
+ expect(
+ readFileSync(output, "utf8")
+ .trimEnd()
+ .split("\n")
+ .map((row) => JSON.parse(row) as { path: string; preview: string })
+ .map(({ path, preview }) => ({ path, preview })),
+ ).toEqual([{ path: "src/変更.py", preview: "value = 2" }]);
+ expect(probe.status, probe.stderr).toBe(0);
+ const target = JSON.parse(probe.stdout) as {
+ root: string;
+ pathspec: string;
+ subject: string;
+ diff: { kind: string; baseRevision: string; headRevision: string };
+ };
+ expect(basename(target.root)).toBe("repository-漢字");
+ expect(target).toMatchObject({
+ pathspec: ".",
+ subject: "café",
+ diff: { kind: "commit", baseRevision: head, headRevision: legacyHead },
+ });
+});
diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts
index d27a21162..7dbf80c8f 100644
--- a/sdk/typescript/tests-ts/runtime.test.ts
+++ b/sdk/typescript/tests-ts/runtime.test.ts
@@ -2028,6 +2028,7 @@ describe("plugin runtime preparation", () => {
expect(workerEnvironment["CODEX_CLI_PATH"]).toBe(
resolveCodexCommand().command,
);
+ expect(workerEnvironment["PYTHONUTF8"]).toBe("1");
const globalCodex = spawnSync("codex", ["--version"], {
encoding: "utf8",
env: workerEnvironment,
@@ -2059,6 +2060,7 @@ describe("plugin runtime preparation", () => {
CODEX_CLI_PATH: configured,
PATH: "",
PYTHON: "/managed/python",
+ PYTHONUTF8: "1",
});
expect(
pluginExecutionEnvironment("/managed/python", {
@@ -3928,7 +3930,7 @@ describe("runtime directories and plugin Python boundary", () => {
"assert os.environ.get('OPENROUTER_API_KEY') is None",
"assert os.environ.get('FIREWORKS_API_KEY') is None",
"payload = sys.stdin.read()",
- "print(json.dumps({'ok': True, 'inputLength': len(payload), 'details': 'x' * (5 * 1024 * 1024)}))",
+ "print(json.dumps({'ok': True, 'label': '出力', 'inputLength': len(payload), 'details': 'x' * (5 * 1024 * 1024)}, ensure_ascii=False))",
].join("\n"),
);
const python = Bun.which("python3") ?? Bun.which("python");
@@ -3949,6 +3951,7 @@ describe("runtime directories and plugin Python boundary", () => {
"x".repeat(64 * 1024),
);
expect(result["ok"]).toBe(true);
+ expect(result["label"]).toBe("出力");
expect(result["inputLength"]).toBe(64 * 1024);
expect(result["details"]).toHaveLength(5 * 1024 * 1024);
});
@@ -4892,6 +4895,48 @@ describe("runtime directories and plugin Python boundary", () => {
).toBe(await realpath(interpreter!));
});
+ test.skipIf(process.platform !== "win32")(
+ "runs plugin helpers with UTF-8 standard streams",
+ async () => {
+ const root = await temporaryDirectory("codex-security-python-utf8-");
+ const repository = join(root, "repository");
+ const output = join(root, "出力.jsonl");
+ await mkdir(repository);
+ await writeFile(join(repository, "source.py"), "value = 1\n");
+ const python = Bun.which("python3") ?? Bun.which("python");
+ expect(python).not.toBeNull();
+
+ const result = spawnSync(
+ python!,
+ [
+ "-B",
+ join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),
+ "make-repo-rank-input",
+ "--repo",
+ repository,
+ "--out",
+ output,
+ ],
+ {
+ encoding: "utf8",
+ env: pluginExecutionEnvironment(python!, {
+ ...process.env,
+ pythonutf8: "0",
+ }),
+ },
+ );
+
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain("出力.jsonl");
+ expect(
+ (await readFile(output, "utf8"))
+ .trimEnd()
+ .split("\n")
+ .map((row) => (JSON.parse(row) as { path: string }).path),
+ ).toEqual(["source.py"]);
+ },
+ );
+
testPosix("uses configured, inherited, and managed Python", async () => {
const root = await temporaryDirectory();
const configured = join(root, "configured-python");
@@ -4938,6 +4983,7 @@ describe("runtime directories and plugin Python boundary", () => {
expect(pluginExecutionEnvironment(managed, { TEST: "1" })).toEqual({
TEST: "1",
PYTHON: managed,
+ PYTHONUTF8: "1",
CODEX_CLI_PATH: resolveCodexCommand().command,
});
await expect(
diff --git a/sdk/typescript/tests-ts/workbench-nested-git-utf8.test.ts b/sdk/typescript/tests-ts/workbench-nested-git-utf8.test.ts
new file mode 100644
index 000000000..8cab07e2a
--- /dev/null
+++ b/sdk/typescript/tests-ts/workbench-nested-git-utf8.test.ts
@@ -0,0 +1,83 @@
+import { spawnSync } from "node:child_process";
+import {
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "bun:test";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+
+const temporaryDirectories: string[] = [];
+
+afterEach(() => {
+ for (const directory of temporaryDirectories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true });
+ }
+});
+
+function git(directory: string, ...args: string[]): void {
+ const result = spawnSync("git", ["-C", directory, ...args], {
+ encoding: "utf8",
+ windowsHide: true,
+ });
+ expect(result.status, result.stderr).toBe(0);
+}
+
+function pythonExecutable(): string | null {
+ return (
+ process.env["PYTHON"] ??
+ Bun.which("python3") ??
+ Bun.which("python") ??
+ Bun.which("py")
+ );
+}
+
+test("writes nested Git pointers as UTF-8 independently of the locale", () => {
+ const root = realpathSync(
+ mkdtempSync(join(tmpdir(), "codex-security-nested-git-utf8-")),
+ );
+ temporaryDirectories.push(root);
+ const repository = join(root, "repository");
+ const nested = join(repository, "nested-漢字");
+ const checkout = join(root, "checkout");
+ mkdirSync(nested, { recursive: true });
+ git(repository, "init", "-q");
+ git(nested, "init", "-q");
+
+ const python = pythonExecutable();
+ expect(python).not.toBeNull();
+ const probe = [
+ "import pathlib, sys",
+ "sys.path.insert(0, sys.argv[1])",
+ "import workbench_target as target",
+ "original_open = pathlib.Path.open",
+ "def locale_open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None):",
+ " if 'b' not in mode and encoding is None:",
+ " encoding = 'cp1252'",
+ " return original_open(self, mode, buffering, encoding, errors, newline)",
+ "pathlib.Path.open = locale_open",
+ "target.copy_git_worktree_files(pathlib.Path(sys.argv[2]), pathlib.Path(sys.argv[3]), ())",
+ ].join("\n");
+ const result = spawnSync(
+ python!,
+ [
+ "-I",
+ "-B",
+ "-c",
+ probe,
+ join(PLUGIN_ROOT, "scripts"),
+ repository,
+ checkout,
+ ],
+ { encoding: "utf8", windowsHide: true },
+ );
+
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(join(checkout, "nested-漢字", ".git"), "utf8")).toContain(
+ "nested-漢字",
+ );
+});