OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(inventory): preserve filename bytes and Unicode owner evidence (#1509)
mldangelo-oaiec4b5b5efb10
diff --git a/plugins/codex-security/mcp-app/src/artifact-inventory.ts b/plugins/codex-security/mcp-app/src/artifact-inventory.tsindex c71452f89..4852f1491 100644--- a/plugins/codex-security/mcp-app/src/artifact-inventory.ts+++ b/plugins/codex-security/mcp-app/src/artifact-inventory.ts@@ -1,4 +1,5 @@import { execFile as nodeExecFile } from "node:child_process";+import { readFile } from "node:fs/promises";import { join } from "node:path";import { promisify } from "node:util";import * as z from "zod/v4";@@ -6,8 +7,8 @@ import commonSchema from "../../schemas/definitions/artifact-common.schema.json"import reviewItemsSchema from "../../schemas/tools/review-items.schema.json";import {artifactDestination,+ artifactSourcePath,paginateArtifactRows,- readArtifactText,type ArtifactContext,type ArtifactPage,} from "./artifact-io.js";@@ -19,6 +20,7 @@ import {missingPythonHelperMessage,resolvePythonCommand,} from "./python_command.js";+import { decodeUtf8 } from "./helpers/utf8.js";const execFile = promisify(nodeExecFile);const documents = [commonSchema, reviewItemsSchema] as SchemaDocument[];@@ -150,7 +152,14 @@ export async function listCodexSecurityReviewItems(async function readReviewItems(context: ArtifactContext,): Promise<ReviewItem[]> {- const source = await readArtifactText(context, inventoryComponents, label);+ const path = await artifactSourcePath(context, inventoryComponents, label);+ const contents = await readFile(path).catch((error: unknown) => {+ throw new Error(+ `${label}: the requested artifact cannot be read: ${error instanceof Error ? error.message : String(error)}`,+ { cause: error },+ );+ });+ const source = decodeUtf8(contents);const rows: ReviewItem[] = [];for (const [index, line] of source.split(/\r?\n/u).entries()) {diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.tsindex c91869aaa..e5bf77567 100644--- a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts+++ b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts@@ -1,7 +1,14 @@import { createTemporaryDirectories } from "./support/temporary-directories.ts";import assert from "node:assert/strict";import { execFile as nodeExecFile } from "node:child_process";-import { mkdir, readFile, symlink, unlink, writeFile } from "node:fs/promises";+import {+ chmod,+ mkdir,+ readFile,+ symlink,+ unlink,+ writeFile,+} from "node:fs/promises";import path from "node:path";import { promisify } from "node:util";import { importSource } from "./import-module.ts";@@ -16,6 +23,8 @@ const inventory = await importSource(try {await testSchemasAreBoundAndExact();await testPrepareUsesTheExistingStandardGenerator();+ if (!["win32", "darwin"].includes(process.platform))+ await testNonUtf8InventoryPaths();await testPrepareListsIgnoredTrackedFilesOnce();await testPrepareExcludesGitMetadata();await testPrepareUsesOnlyAuthoritativeDiffChanges();@@ -25,7 +34,7 @@ try {await testEmptyInventoryIsValid();await testUnsafeInventoryRowsAreRejected();await testSymlinkedInventoryIsRejected();- await testMissingInventoryIsReported();+ await testInventoryReadErrorsAreReported();await testWorkersCannotPrepareInventory();await testBoundScopeFailurePreservesPreviousInventory();await testPythonLaunchErrors();@@ -60,6 +69,8 @@ async function testPrepareUsesTheExistingStandardGenerator() {const fixture = await createFixture("standard repository");await fixture.writeRepositoryFile("src/a.ts", "export const a = 1;\n");await fixture.writeRepositoryFile("src/résumé.ts", "export const b = 2;\n");+ for (const name of ["\uFEFF来源.ts", "name-\uFFFD.ts", "🙂.ts"])+ await fixture.writeRepositoryFile(`src/${name}`, "export {};\n");await fixture.writeRepositoryFile(".hidden/handler.ts","export const c = 3;\n",@@ -117,6 +128,36 @@ async function testPrepareUsesTheExistingStandardGenerator() {);}+async function testNonUtf8InventoryPaths() {+ const fixture = await createFixture("non-UTF-8 repository names");+ const rawPath = Buffer.concat([+ Buffer.from(path.join(fixture.repoRoot, "name-")),+ Buffer.from([0xff]),+ Buffer.from(".ts"),+ ]);+ await writeFile(rawPath, "raw path\n");+ await fixture.writeRepositoryFile("name-�.ts", "Unicode path\n");+ await assert.rejects(+ inventory.prepareCodexSecurityReviewItems(fixture.scan),+ /not valid for encoding utf-8/,+ );+ assert.ok((await readFile(fixture.scanInventory)).includes(0xff));+ await assert.rejects(+ inventory.listCodexSecurityReviewItems(fixture.scan),+ /not valid for encoding utf-8/,+ );+ await unlink(rawPath);+ assert.deepEqual(+ await inventory.prepareCodexSecurityReviewItems(fixture.scan),+ {+ reviewItemsTotal: 1,+ },+ );+ assert.deepEqual(await inventory.listCodexSecurityReviewItems(fixture.scan), {+ items: [{ path: "./name-�.ts" }],+ });+}+async function testPrepareListsIgnoredTrackedFilesOnce() {const fixture = await createFixture("ignored tracked files");await runGit(fixture.repoRoot, "init", "-q");@@ -374,13 +415,38 @@ async function testSymlinkedInventoryIsRejected() {);}-async function testMissingInventoryIsReported() {+async function testInventoryReadErrorsAreReported() {const fixture = await createFixture("missing inventory");await assert.rejects(inventory.listCodexSecurityReviewItems(fixture.scan),/review_items.*(?:unavailable|missing|read)/i,);+ if (process.platform !== "win32" && process.getuid?.() !== 0) {+ await writeInventory(fixture.scanInventory, "./src/a.ts\n");+ try {+ await chmod(fixture.scanInventory, 0o000);+ await assert.rejects(+ inventory.listCodexSecurityReviewItems(fixture.scan),+ (error: Error & { cause?: NodeJS.ErrnoException }) => {+ assert.equal(error.cause?.code, "EACCES");+ assert.equal(+ error.message,+ `review_items: the requested artifact cannot be read: ${error.cause.message}`,+ );+ return true;+ },+ );+ } finally {+ await chmod(fixture.scanInventory, 0o600);+ }+ assert.deepEqual(+ await inventory.listCodexSecurityReviewItems(fixture.scan),+ {+ items: [{ path: "./src/a.ts" }],+ },+ );+ }}async function testWorkersCannotPrepareInventory() {diff --git a/sdk/typescript/src/component-plan.ts b/sdk/typescript/src/component-plan.tsindex 099f072a7..e5ff9209a 100644--- a/sdk/typescript/src/component-plan.ts+++ b/sdk/typescript/src/component-plan.ts@@ -269,6 +269,7 @@ async function inventoryFiles(signal?: AbortSignal,): Promise<string[]> {signal?.throwIfAborted();+ const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });if (await enclosingGitWorktreeRoot(repository, signal)) {validatedGitEnvironment();const git = await resolveTrustedExecutable("git", process.env, repository);@@ -288,15 +289,19 @@ async function inventoryFiles("--",".",],- { env: git.environment, signal, maxBuffer: Infinity },+ { env: git.environment, signal, maxBuffer: Infinity, encoding: "buffer" },);const files: string[] = [];- for (const path of stdout.split("\0").filter(Boolean)) {+ for (const path of stdout.toString("latin1").split("\0").filter(Boolean)) {signal?.throwIfAborted();- const metadata = await lstat(join(repository, path)).catch(- nullIfMissingFile,- );- if (metadata?.isFile()) files.push(join(repository, path));+ const bytes = Buffer.from(path, "latin1");+ const metadata = await lstat(+ Buffer.from(+ join(Buffer.from(repository).toString("latin1"), path),+ "latin1",+ ),+ ).catch(nullIfMissingFile);+ if (metadata?.isFile()) files.push(join(repository, utf8.decode(bytes)));}return files.length === 0? []@@ -307,11 +312,27 @@ async function inventoryFiles(while (pending.length > 0) {signal?.throwIfAborted();const directory = pending.pop()!;- for (const entry of await readdir(join(repository, directory), {- withFileTypes: true,- })) {- if (entry.name === ".git") continue;- const path = directory ? `${directory}/${entry.name}` : entry.name;+ const directoryPath = join(repository, directory);+ // Bun's buffer encoding omits Dirent metadata even with withFileTypes.+ const entries = process.versions["bun"]+ ? await readdir(directoryPath, {+ withFileTypes: true,+ encoding: process.platform === "win32" ? "utf8" : "latin1",+ })+ : await readdir(directoryPath, {+ withFileTypes: true,+ encoding: "buffer",+ });+ for (const entry of entries) {+ if (!entry.isDirectory() && !entry.isFile()) continue;+ const name =+ typeof entry.name === "string"+ ? process.platform === "win32"+ ? entry.name+ : utf8.decode(Buffer.from(entry.name, "latin1"))+ : utf8.decode(entry.name);+ if (name === ".git") continue;+ const path = directory ? `${directory}/${name}` : name;if (entry.isDirectory()) pending.push(path);else if (entry.isFile()) files.push(path);}diff --git a/sdk/typescript/src/owner-evidence.ts b/sdk/typescript/src/owner-evidence.tsindex df4a63333..4beeb1e1d 100644--- a/sdk/typescript/src/owner-evidence.ts+++ b/sdk/typescript/src/owner-evidence.ts@@ -57,15 +57,16 @@ export async function ownerRepository();if (executable === null)throw new CodexSecurityError("Git is required to suggest owners.");- const run = async (cwd: string, args: string[]) => {+ const run = async (cwd: string, args: string[], encoding: BufferEncoding) => {signal?.throwIfAborted();const { stdout } = await execFile(executable.executable, args, {cwd,env: { ...executable.environment, GIT_NO_LAZY_FETCH: "1" },signal,maxBuffer: Infinity,+ encoding: "buffer",});- return stdout;+ return stdout.toString(encoding);};const [gitDirectory, commonDirectory, ...objectDirectories] =await gitMetadataDirectories(repository, signal, {@@ -85,24 +86,34 @@ export async function ownerRepository(}await requireBoundMetadata(metadataDirectories, objectDirectories, signal);// Blame otherwise reads an uncommitted .mailmap from the working directory.- const git = (...args: string[]) =>- run(gitDirectory, [- `--git-dir=${gitDirectory}`,- "--literal-pathspecs",- "-c",- "core.bare=true",- ...args,- ]);+ const gitOutput = (encoding: BufferEncoding, ...args: string[]) =>+ run(+ gitDirectory,+ [+ `--git-dir=${gitDirectory}`,+ "--literal-pathspecs",+ "-c",+ "core.bare=true",+ "-c",+ "i18n.logOutputEncoding=UTF-8",+ ...args,+ ],+ encoding,+ );+ const git = (...args: string[]) => gitOutput("utf8", ...args);const revision = (await git("rev-parse", "--verify", "HEAD^{commit}")).trim();const files = new Set(- (await git("ls-tree", "-r", "-z", "--full-tree", revision))+ (await gitOutput("latin1", "ls-tree", "-r", "-z", "--full-tree", revision)).split("\0").filter((record) => /^100(?:644|755) blob /u.test(record)).map((record) => record.slice(record.indexOf("\t") + 1)),);const shallow =(await git("rev-parse", "--is-shallow-repository")).trim() === "true";- return { git, revision, files, shallow };+ // Latin-1 keys preserve Git's filename bytes, including unrelated non-UTF-8 names.+ const hasFile = (path: string) =>+ path.isWellFormed() && files.has(Buffer.from(path).toString("latin1"));+ return { git, revision, hasFile, shallow };}/** Neither references nor objects may link into another checkout. */@@ -148,7 +159,7 @@ export async function collectOwnerEvidence(finding: OwnerFinding,repository: Awaited<ReturnType<typeof ownerRepository>>,): Promise<OwnerContext> {- const { git, revision, files, shallow } = repository;+ const { git, revision, hasFile, shallow } = repository;const context: OwnerContext = {identities: [],evidence: [],@@ -179,7 +190,7 @@ export async function collectOwnerEvidence(context.evidence.push({ id: `e${context.evidence.length + 1}`, ...item });};for (const path of new Set(finding.locations.map(({ path }) => path))) {- if (!files.has(path)) {+ if (!hasFile(path)) {context.limitations.push(`Not a regular file at HEAD: ${path}`);continue;}diff --git a/sdk/typescript/tests-ts/component-scan.test.ts b/sdk/typescript/tests-ts/component-scan.test.tsindex 4f86fa4ad..d51667d7f 100644--- a/sdk/typescript/tests-ts/component-scan.test.ts+++ b/sdk/typescript/tests-ts/component-scan.test.ts@@ -1,7 +1,8 @@import { codexWithRun } from "./support/codex.js";import { createCliTest } from "./support/cli-run.js";-import { gitText } from "./support/shell.js";+import { gitText, nodeCommand } from "./support/shell.js";import { resolving } from "./support/promises.js";+import { runTestInSubprocess } from "./support/test-subprocess.js";import { execFileSync } from "node:child_process";import {mkdir,@@ -11,9 +12,11 @@ import {rename,stat,symlink,+ unlink,writeFile,} from "node:fs/promises";import { dirname, join, relative, sep } from "node:path";+import { fileURLToPath, pathToFileURL } from "node:url";import { afterEach, expect, spyOn, test, mock } from "bun:test";import type { ScanOptions } from "../src/api.js";import { writeThreatModel } from "../src/artifact-export.js";@@ -987,6 +990,100 @@ test("plans from a Git inventory without tools or ignored files", async () => {}});+// Windows and macOS do not support the synthetic 0xff filename fixture.+for (const rawByteNames of [false, true]) {+ test+ .skipIf(rawByteNames && ["win32", "darwin"].includes(process.platform))+ .each(["Git", "plain directory"])(+ `preserves exact UTF-8 paths in a %s component inventory${rawByteNames ? " with raw-byte entries" : ""}`,+ async (kind) => {+ const repository = join(await temporaryDirectory(), "repository");+ await mkdir(repository);+ if (kind === "Git") execFileSync("git", ["-C", repository, "init", "-q"]);+ const paths = ["name-\uFFFD.ts", "\uFEFF来源.ts", "résumé.ts", "🙂.ts"];+ for (const path of paths)+ await writeFile(join(repository, path), "export {};\n");+ const proposed = { components: [{ name: "Sources", paths }] };+ const response = mock(() => proposed);+ const options = { codex: fakeCodex(response) };+ if (rawByteNames) {+ const invalid = Buffer.concat([+ Buffer.from(join(repository, "name-")),+ Buffer.from([0xff]),+ Buffer.from(".ts"),+ ]);+ await writeFile(invalid, "export {};\n");+ await expect(+ planComponents(repository, options),+ ).rejects.toBeInstanceOf(TypeError);+ expect(response).not.toHaveBeenCalled();+ await unlink(invalid);+ await symlink("missing.ts", invalid);+ }+ expect(await planComponents(repository, options)).toEqual(proposed);+ expect(response).toHaveBeenCalledTimes(1);+ },+ );+}++test("plans Unicode files when directory entry types are unknown", async () => {+ if (+ runTestInSubprocess(+ import.meta.path,+ "plans Unicode files when directory entry types are unknown",+ )+ )+ return;+ const root = await temporaryDirectory();+ const repository = join(root, "repository");+ const paths = ["name-\uFFFD.ts", "\uFEFF来源.ts", "résumé/🙂.ts"];+ for (const path of [...paths, ".git/ignored.ts"]) {+ await mkdir(dirname(join(repository, path)), { recursive: true });+ await writeFile(join(repository, path), "export {};\n");+ }+ const source = new URL("../src/component-plan.ts", import.meta.url);+ const built = await Bun.build({+ entrypoints: [fileURLToPath(source)],+ target: "node",+ format: "esm",+ define: { "import.meta.url": JSON.stringify(source.href) },+ });+ expect(built.success).toBe(true);+ const module = join(root, "component-plan.mjs");+ await writeFile(module, await built.outputs[0]!.text());+ const proposed = { components: [{ name: "Sources", paths }] };+ const output = execFileSync(+ nodeCommand().command,+ [+ "--input-type=module",+ "--eval",+ `+ import assert from "node:assert/strict";+ const binding = process.binding("fs");+ const original = binding.readdir;+ let unknownEntries = 0;+ binding.readdir = async function (...args) {+ const result = await original.apply(this, args);+ if (args[2] === true) {+ unknownEntries += result[1].length;+ result[1].fill(0);+ }+ return result;+ };+ const { planComponents } = await import(${JSON.stringify(pathToFileURL(module).href)});+ const proposed = ${JSON.stringify(proposed)};+ const plan = await planComponents(${JSON.stringify(repository)}, {+ codex: { startThread: () => ({ run: async () => ({ finalResponse: JSON.stringify(proposed) }) }) },+ });+ assert.ok(unknownEntries > 0);+ console.log(JSON.stringify(plan));+ `,+ ],+ { encoding: "utf8" },+ );+ expect(JSON.parse(output)).toEqual(proposed);+});+test.each(["directories", "manifests", "root files"])("batches oversized %s without dropping inventory paths",(layout) => {diff --git a/sdk/typescript/tests-ts/suggest-owners.test.ts b/sdk/typescript/tests-ts/suggest-owners.test.tsindex cab02606d..20887f900 100644--- a/sdk/typescript/tests-ts/suggest-owners.test.ts+++ b/sdk/typescript/tests-ts/suggest-owners.test.ts@@ -1,7 +1,7 @@import { modelResponseText } from "./support/model-response-text.js";import { execFile as execFileCallback } from "node:child_process";import { cp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises";-import { join, relative } from "node:path";+import { delimiter, join, relative } from "node:path";import { promisify } from "node:util";import type { ThreadOptions, TurnOptions } from "@openai/codex-sdk";import { afterEach, expect, test } from "bun:test";@@ -94,6 +94,65 @@ function chooseAlex(context: OwnerContext) {};}+test.each(["ISO-8859-1", "UTF-16LE"])(+ "preserves blame and history identities with %s Git output configured",+ async (encoding) => {+ const repo = await repository();+ for (const [name, email, version, subject] of [+ ["Renée Example", "[email protected]", 1, "Ancienne révision"],+ ["Zoë Example", "[email protected]", 2, "Nouvelle révision"],+ ] as const) {+ await writeFile(+ join(repo.path, "handler.ts"),+ `// façade\nexport const version = ${version};\n`,+ );+ await repo.git(+ "-c",+ `user.name=${name}`,+ "-c",+ `user.email=${email}`,+ "commit",+ "-qam",+ subject,+ );+ }+ await repo.git("config", "i18n.logOutputEncoding", encoding);+ for (const [name, email] of [+ ["Renée Example", "[email protected]"],+ ["Zoë Example", "[email protected]"],+ ]) {+ const { codex, calls } = fakeCodex((context) => {+ const identityIndex = context.identities.findIndex(+ (identity) => identity.email === email,+ );+ return {+ identityIndex,+ reason: "Synthetic ownership selection.",+ evidenceIds: context.evidence+ .filter((item) => item.identityIndex === identityIndex)+ .map(({ id }) => id),+ };+ });+ const report = await suggestOwners(repo.path, [finding], { codex });+ expect(report.results[0]).toMatchObject({+ status: "identified",+ owner: { name, email },+ });+ const { context } = calls[0]!;+ expect(+ context.evidence.some((item) => item.content.includes("façade")),+ ).toBe(true);+ for (const subject of ["Ancienne révision", "Nouvelle révision"])+ expect(+ context.evidence.some((item) => item.content.endsWith(subject)),+ ).toBe(true);+ }+ expect(await repo.git("config", "--get", "i18n.logOutputEncoding")).toBe(+ encoding,+ );+ },+);+test.each(["borrowed-gitfile", "forged-commondir"])("rejects %s before sending another checkout's evidence to Codex",async (kind) => {@@ -348,6 +407,85 @@ test("abstains without a model call when locations do not identify committed reg).toBe(true);});+// Keep Unicode coverage even where raw-byte fixture names are unsupported.+for (const rawByteNames of [false, true]) {+ test.skipIf(rawByteNames && ["win32", "darwin"].includes(process.platform))(+ `matches committed ${rawByteNames ? "raw-byte" : "Unicode"} filenames without aliasing replacement characters`,+ async () => {+ const repo = await repository();+ const path = "name-�.ts";+ if (rawByteNames) {+ await writeFile(+ Buffer.concat([+ Buffer.from(join(repo.path, "name-")),+ Buffer.from([0xff]),+ Buffer.from(".ts"),+ ]),+ "export const unrelated = 1;\n",+ );+ await repo.git("add", ".");+ await repo.git("commit", "-qm", "Add raw-byte filename");+ }+ const { codex, calls } = fakeCodex();+ const input = [{ ...finding, locations: [{ path, startLine: 1 }] }];+ const missing = await suggestOwners(repo.path, input, { codex });+ expect(missing.results[0]!.status).toBe("abstained");+ expect(missing.results[0]!.limitations).toContain(+ `Not a regular file at HEAD: ${path}`,+ );+ expect(calls).toHaveLength(0);++ await writeFile(+ join(repo.path, path),+ "export const actualUnicodeFile = 2;\n",+ );+ await repo.git("add", ".");+ await repo.git("commit", "-qm", "Add Unicode filename");+ const report = await suggestOwners(repo.path, input, { codex });+ expect(report.results[0]!.status).toBe("identified");+ expect(+ calls[0]!.context.evidence.find(({ kind }) => kind === "source")!+ .content,+ ).toBe("1: export const actualUnicodeFile = 2;");+ const malformed = await suggestOwners(+ repo.path,+ [{ ...finding, locations: [{ path: "name-\ud800.ts" }] }],+ { codex },+ );+ expect(malformed.results[0]!.status).toBe("abstained");+ expect(calls).toHaveLength(1);++ if (process.platform === "win32") return;+ const tools = await temporaryDirectory();+ const diagnostic = "Permission denied: café/東/😀";+ await writeFile(+ join(tools, "git"),+ `#!/bin/sh+for argument; do+ if [ "$argument" = ls-tree ]; then+ printf '%s\\n' "$SYNTHETIC_GIT_DIAGNOSTIC" >&2+ exit 13+ fi+done+exec "$SYNTHETIC_REAL_GIT" "$@"+`,+ { mode: 0o700 },+ );+ await expect(+ suggestOwners(repo.path, input, {+ codex,+ environment: {+ ...process.env,+ PATH: `${tools}${delimiter}${process.env["PATH"] ?? ""}`,+ SYNTHETIC_REAL_GIT: Bun.which("git")!,+ SYNTHETIC_GIT_DIAGNOSTIC: diagnostic,+ },+ }),+ ).rejects.toThrow(diagnostic);+ },+ );+}+test("keeps renamed-file blame citations at HEAD and inherits SDK model settings", async () => {const repo = await repository();const path = "record [1].ts";