Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(mcp): use UTF-8 for workbench responses and diagnostics (#2566)

Michael D'Angelo34d0e634ea0d
diff --git a/plugins/codex-security/mcp-app/src/server/workbench-process.ts b/plugins/codex-security/mcp-app/src/server/workbench-process.ts
index 7a62de447..8e00f0d11 100644
--- a/plugins/codex-security/mcp-app/src/server/workbench-process.ts
+++ b/plugins/codex-security/mcp-app/src/server/workbench-process.ts
@@ -1,6 +1,8 @@
export const WORKBENCH_PYTHON = `
import json, sys
from pathlib import Path
+sys.stdout.reconfigure(encoding="utf-8", errors=sys.stdout.errors)
+sys.stderr.reconfigure(encoding="utf-8", errors=sys.stderr.errors)
script = Path(sys.argv.pop(1)).resolve()
sys.path.insert(0, str(script.parent))
args = json.loads(sys.stdin.buffer.readline())
diff --git a/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts b/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts
index e5f1dd08a..9e2a92cae 100644
--- a/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts
+++ b/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts
@@ -71,7 +71,7 @@ const profileContents = profileIndex === -1 ? undefined : readFileSync(join(proc
const openaiAuthentication = stdin.includes('CAPTURE_SYNTHETIC_OPENAI_AUTH') ? { OPENAI_API_KEY: process.env.OPENAI_API_KEY, CODEX_API_KEY: process.env.CODEX_API_KEY } : undefined;
const bedrockAuthentication = stdin.includes('CAPTURE_SYNTHETIC_BEDROCK_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_BEDROCK_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined;
const codexConfig = stdin.includes('CAPTURE_SYNTHETIC_CODEX_CONFIG') ? readFileSync(join(process.env.CODEX_HOME, 'config.toml'), 'utf8') : undefined;
-const runtimeEnvironment = Object.fromEntries(['PATH', 'HOME', 'PYTHON', 'PYTHONUTF8', 'CODEX_MCP_NODE_PATH', 'CODEX_BROWSER_USE_NODE_PATH', 'CODEX_ELECTRON_RESOURCES_PATH', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'DYLD_FALLBACK_LIBRARY_PATH', 'CODEX_SECURITY_STATE_DIR', 'RUNNER_TRACKING_ID'].map(name => [name, process.env[name]]));
+const runtimeEnvironment = Object.fromEntries(['PATH', 'HOME', 'PYTHON', 'PYTHONUTF8', 'PYTHONIOENCODING', 'CODEX_MCP_NODE_PATH', 'CODEX_BROWSER_USE_NODE_PATH', 'CODEX_ELECTRON_RESOURCES_PATH', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'DYLD_FALLBACK_LIBRARY_PATH', 'CODEX_SECURITY_STATE_DIR', 'RUNNER_TRACKING_ID'].map(name => [name, process.env[name]]));
const toolProbe = stdin.includes('CAPTURE_SYNTHETIC_RG') ? spawnSync('rg', ['--version'], { encoding: 'utf8' }) : undefined;
if (toolProbe && toolProbe.status !== 0) throw toolProbe.error ?? new Error(toolProbe.stderr);
const pythonProbe = stdin.includes('CAPTURE_SYNTHETIC_PYTHON') ? spawnSync(process.env.PYTHON, ['-I', '-c', 'import json,os,sys; print(json.dumps([sys.prefix,os.environ.get("LD_LIBRARY_PATH")]))'], { encoding: 'utf8' }) : undefined;
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
index c6ce61d45..0ce4f8508 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
@@ -1446,6 +1446,7 @@ async function testWorkerRuntimeSettings() {
];
const saved = [
"PYTHON",
+ "PYTHONIOENCODING",
"CODEX_SECURITY_PYTHON_COMMAND",
"PATH",
"HOME",
@@ -1717,6 +1718,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
};
return {
nativePaths,
+ pythonIoEncoding: index % 2 === 0 ? "utf-16-le" : "utf-8-sig",
knowledgePath: path.join(fixture.root, `knowledge-${index}`),
knowledgeDocuments: {
"1-architecture.md.txt": `Synthetic architecture for scan ${index}.`,
@@ -2125,6 +2127,8 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
]) {
process.env[name] = path.join(fixture.root, `libraries-${index}`);
}
+ process.env.PYTHONIOENCODING =
+ workerConfigurations[index].pythonIoEncoding;
restoreEnv(
"CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH",
workerConfigurations[index].workerConfigPath,
@@ -2248,6 +2252,10 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
workerLaunch.environment![name],
);
}
+ assert.equal(
+ invocation.runtimeEnvironment.PYTHONIOENCODING,
+ workerConfigurations[index].pythonIoEncoding,
+ );
assert.equal(
invocation.knowledgePath,
workerConfigurations[index].knowledgePath,
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_state.ts b/plugins/codex-security/mcp-app/tests/test_workbench_state.ts
index e3c2b2730..a3c5a008b 100644
--- a/plugins/codex-security/mcp-app/tests/test_workbench_state.ts
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_state.ts
@@ -97,7 +97,10 @@ try {
const diagnostic = direct.stderr.trim().split("\n").at(-1)!;
assert.match(diagnostic, /FileExistsError/);
assert.equal(wrapped.stderr.trim().split("\n").at(-1), diagnostic);
- const failure = await openWorkspace(environment);
+ const failure = await openWorkspace({
+ ...environment,
+ PYTHONIOENCODING: "utf-16-le",
+ });
assert.equal(failure.error, undefined);
assert.equal(failure.result.isError, true);
assert.ok(
@@ -111,7 +114,12 @@ try {
...environment,
CODEX_SECURITY_STATE_DIR: explicitState,
};
- assertNoError(await openWorkspace(explicitEnvironment));
+ assertNoError(
+ await openWorkspace({
+ ...explicitEnvironment,
+ PYTHONIOENCODING: "utf-8-sig",
+ }),
+ );
assert.equal(
(await stat(path.join(explicitState, "workbench.sqlite3"))).isFile(),
true,