Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(targets): preserve Git checkout formats and raw POSIX inventory paths (#1288)

Michael D'Angeloe1adad147a88
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index 669acae8b..c96b833aa 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -421,7 +421,7 @@ jobs:
python -m pytest plugins/codex-security/tests -m "$PYTEST_MARKERS" -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml
- name: Require native platform coverage
if: runner.os != 'Linux'
- run: python -c "import xml.etree.ElementTree as ET; assert not ET.parse('reports/python.xml').findall('.//testcase/skipped'), 'Native platform tests must run without skips'"
+ run: python -c "import pathlib, xml.etree.ElementTree as ET; assert not any(ET.parse(report).findall('.//testcase/skipped') for report in pathlib.Path('reports').glob('python*.xml')), 'Native platform tests must run without skips'"
- name: Upload Python test reports
if: always()
continue-on-error: true
@@ -429,7 +429,7 @@ jobs:
with:
name: python-test-reports-${{ matrix.os }}-${{ matrix.python }}
overwrite: true
- path: reports/python.xml
+ path: reports/python*.xml
if-no-files-found: warn
retention-days: 14
- name: Set up Node.js for triage evals
diff --git a/plugins/codex-security/mcp-app/src/artifact-discovery.ts b/plugins/codex-security/mcp-app/src/artifact-discovery.ts
index d04a99668..0d736551c 100644
--- a/plugins/codex-security/mcp-app/src/artifact-discovery.ts
+++ b/plugins/codex-security/mcp-app/src/artifact-discovery.ts
@@ -3,6 +3,7 @@ import { createRequire } from "node:module";
import { join } from "node:path";
import { Worker } from "node:worker_threads";
import { decodeUtf8 } from "./helpers/utf8.js";
+import { decodePosixBytes } from "./helpers/posix-path.js";
import { resolvePythonCommand, runPythonWithInput } from "./python_command.js";
import {
stableJson,
@@ -161,7 +162,12 @@ async function diffCandidateSources(
}
const nativePath = (value: string) =>
process.platform === "win32" ? value.replaceAll("\\", "/") : value;
- const paths = decodeUtf8(await readFile(inventory))
+ const contents = await readFile(inventory);
+ const paths = (
+ process.platform === "win32"
+ ? decodeUtf8(contents)
+ : decodePosixBytes(contents)
+ )
.split(/\r?\n/u)
.filter(Boolean)
.map(nativePath);
diff --git a/plugins/codex-security/mcp-app/src/artifact-inventory.ts b/plugins/codex-security/mcp-app/src/artifact-inventory.ts
index 4852f1491..23d92a107 100644
--- a/plugins/codex-security/mcp-app/src/artifact-inventory.ts
+++ b/plugins/codex-security/mcp-app/src/artifact-inventory.ts
@@ -21,6 +21,7 @@ import {
resolvePythonCommand,
} from "./python_command.js";
import { decodeUtf8 } from "./helpers/utf8.js";
+import { decodePosixBytes } from "./helpers/posix-path.js";
const execFile = promisify(nodeExecFile);
const documents = [commonSchema, reviewItemsSchema] as SchemaDocument[];
@@ -159,7 +160,10 @@ async function readReviewItems(
{ cause: error },
);
});
- const source = decodeUtf8(contents);
+ const source =
+ process.platform === "win32"
+ ? decodeUtf8(contents)
+ : decodePosixBytes(contents);
const rows: ReviewItem[] = [];
for (const [index, line] of source.split(/\r?\n/u).entries()) {
diff --git a/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.ts b/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.ts
index 731458293..6b784beb3 100644
--- a/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.ts
@@ -148,9 +148,67 @@ function readScope(
allowMissing: boolean,
sources?: ReadonlyMap<string, CandidateSource>,
): Set<string> {
- const contents = decodeUtf8(readFile(path));
+ const bytes = readFile(path);
+ const contents = windows ? decodeUtf8(bytes) : decodePosixBytes(bytes);
+ const lines = contents.split("\n");
+ const listedRows = new Set(lines);
+ const isScopeFile = (value: string): boolean => {
+ try {
+ relativeFile(value, root, sources);
+ return true;
+ } catch {
+ return false;
+ }
+ };
+ const carriageRows = new Map<string, [boolean, boolean]>();
+ if (!windows) {
+ for (const line of lines) {
+ if (line.endsWith("\r") && line !== "\r") {
+ carriageRows.set(line, [
+ isScopeFile(line),
+ isScopeFile(line.slice(0, -1)),
+ ]);
+ }
+ }
+ }
+ const crlfEvidence = lines.slice(0, -1).some((line) => {
+ if (line === "\r") return true;
+ const paths = carriageRows.get(line);
+ return paths !== undefined && paths[1] && !paths[0];
+ });
+ const literalEvidence = [...carriageRows.values()].some(
+ ([literal, stripped]) => literal && !stripped,
+ );
const scope = new Set<string>();
- for (const [index, line] of contents.split(/\r?\n/u).entries()) {
+ for (const [index, row] of lines.entries()) {
+ let line = row;
+ const finalLiteral = index === lines.length - 1;
+ if (!finalLiteral && (windows || line === "\r")) {
+ if (line.endsWith("\r")) line = line.slice(0, -1);
+ } else if (!finalLiteral && line.endsWith("\r")) {
+ const [literal, stripped] = carriageRows.get(line)!;
+ const trimmed = line.slice(0, -1);
+ if (allowMissing && literal && !stripped && !listedRows.has(trimmed)) {
+ throw new Error(
+ `in-scope file row ${index + 1}: ambiguous carriage-return paths`,
+ );
+ }
+ if (stripped && !literal) {
+ line = trimmed;
+ } else if (stripped && literal) {
+ if (!listedRows.has(trimmed)) {
+ if (crlfEvidence && !literalEvidence) {
+ line = trimmed;
+ } else if (!literalEvidence || crlfEvidence) {
+ throw new Error(
+ `in-scope file row ${index + 1}: ambiguous carriage-return paths`,
+ );
+ }
+ }
+ } else if (!literal && crlfEvidence) {
+ line = trimmed;
+ }
+ }
if (line === "") continue;
try {
scope.add(relativeFile(line, root, sources)[0]);
diff --git a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts
index 6a03e7687..ff9ab1499 100644
--- a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts
+++ b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts
@@ -336,9 +336,20 @@ function registerCompactTool<Input>(
},
async (input, requestContext) => {
const value = await registration.handler(input, requestContext);
+ let structured = true;
+ const text = JSON.stringify(value, (key, item: unknown) => {
+ if (
+ !key.isWellFormed() ||
+ (typeof item === "string" && !item.isWellFormed())
+ )
+ structured = false;
+ return item;
+ });
+ // Codex's native JSON decoder rejects lone surrogates; the JSON text retains
+ // their exact escaped representation, including raw POSIX filename bytes.
return {
- content: [{ type: "text" as const, text: JSON.stringify(value) }],
- structuredContent: value as JsonRecord,
+ content: [{ type: "text" as const, text }],
+ ...(structured ? { structuredContent: value as JsonRecord } : {}),
};
},
);
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts b/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts
index ab6802d3c..8a4e6321b 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts
@@ -407,7 +407,9 @@ async function verifyNormalizerFailuresPreserveOutput(
await writeFile(inventory, Buffer.from([0xff]));
await assert.rejects(
recordCodexSecurityDiscoveryCandidates({ candidates: [] }, context),
- /UTF-8|encoded data/,
+ process.platform === "win32"
+ ? /UTF-8|encoded data/
+ : /in-scope file row 1:.*ENOENT/,
);
assert.equal(await readFile(destination, "utf8"), original);
await writeFile(inventory, scope);
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts
index 86082c9f3..cd746f175 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts
@@ -137,14 +137,18 @@ async function testNonUtf8InventoryPaths() {
]);
await writeFile(rawPath, "raw path\n");
await fixture.writeRepositoryFile("name-�.ts", "Unicode path\n");
- await assert.rejects(
- inventory.prepareCodexSecurityReviewItems(fixture.scan),
- /not valid for encoding utf-8/,
+ assert.deepEqual(
+ await inventory.prepareCodexSecurityReviewItems(fixture.scan),
+ {
+ reviewItemsTotal: 2,
+ },
);
assert.ok((await readFile(fixture.scanInventory)).includes(0xff));
- await assert.rejects(
- inventory.listCodexSecurityReviewItems(fixture.scan),
- /not valid for encoding utf-8/,
+ assert.deepEqual(
+ (await inventory.listCodexSecurityReviewItems(fixture.scan)).items
+ .map((item: { path: string }) => item.path)
+ .sort(),
+ ["./name-\udcff.ts", "./name-�.ts"],
);
await unlink(rawPath);
assert.deepEqual(
diff --git a/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts b/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts
new file mode 100644
index 000000000..a26d8363b
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts
@@ -0,0 +1,171 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { copyFile, mkdir, readFile, unlink, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { test } from "node:test";
+import type { ArtifactContext } from "../src/artifact-io.js";
+import { importSource } from "./import-module.ts";
+import { createTemporaryDirectories } from "./support/temporary-directories.ts";
+
+const { normalizeCandidatesCommand } = await importSource(
+ new URL("../src/helpers/normalize-candidates.ts", import.meta.url).pathname,
+);
+const { recordCodexSecurityDiscoveryCandidates } = await importSource(
+ path.join(import.meta.dirname, "../src/artifact-discovery.ts"),
+ {
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/server.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ },
+);
+const inventoryScript = path.join(
+ import.meta.dirname,
+ "../../scripts/generate_in_scope_files.py",
+);
+
+for (const deleted of [false, true]) {
+ test(
+ `normalizes a generated inventory with a ${deleted ? "deleted" : "present"} raw-byte path`,
+ { skip: process.platform !== "linux" },
+ async () => {
+ const directories = createTemporaryDirectories(true);
+ const root = await directories.create("candidate-inventory-");
+ try {
+ const repo = path.join(root, "repository");
+ await mkdir(repo);
+ const rawName = Buffer.from([0x63, 0x61, 0x66, 0xe9, 0x2e, 0x70, 0x79]);
+ const rawPath = Buffer.concat([Buffer.from(`${repo}/`), rawName]);
+ const normalPath = path.join(repo, "app.py");
+ await writeFile(rawPath, "first\n");
+ await writeFile(normalPath, "first\n");
+ const git = (...args: string[]) =>
+ execFileSync(
+ "git",
+ [
+ "-c",
+ "user.name=Fixture",
+ "-c",
+ ...args,
+ ],
+ { cwd: repo, encoding: "utf8" },
+ ).trim();
+ git("init", "-q");
+ git("add", ".");
+ git("commit", "-qm", "initial");
+ const base = git("rev-parse", "HEAD");
+ if (deleted) await unlink(rawPath);
+ else await writeFile(rawPath, "changed\n");
+ await writeFile(normalPath, "changed\n");
+ git("add", ".");
+ git("commit", "-qm", "change");
+
+ const scope = path.join(root, "in-scope.txt");
+ execFileSync(process.env.PYTHON ?? "python3", [
+ inventoryScript,
+ "--repo",
+ repo,
+ "--scope",
+ ".",
+ "--out",
+ scope,
+ "--diff-base",
+ base,
+ "--diff-head",
+ "HEAD",
+ ]);
+ assert.deepEqual(
+ await readFile(scope),
+ Buffer.concat([Buffer.from("app.py\n"), rawName, Buffer.from("\n")]),
+ );
+ const input = path.join(root, "candidates.jsonl");
+ const output = path.join(root, "combined.jsonl");
+ const candidatePath = deleted ? "app.py" : "caf\udce9.py";
+ await writeFile(
+ input,
+ JSON.stringify({
+ cwe_ids: ["CWE-20"],
+ locations: [{ path: candidatePath, start_line: 1, role: "source" }],
+ summary: "Synthetic candidate",
+ evidence: "Synthetic evidence",
+ }) + "\n",
+ );
+ const args = [
+ "--input",
+ input,
+ "--out",
+ output,
+ "--repo-root",
+ repo,
+ "--in-scope-files",
+ scope,
+ ];
+ if (deleted) {
+ assert.equal(normalizeCandidatesCommand(args), 2);
+ args.push("--allow-missing-in-scope");
+ }
+ assert.equal(normalizeCandidatesCommand(args), 0);
+ const retained = await readFile(output, "utf8");
+ assert.equal(JSON.parse(retained).locations[0].path, candidatePath);
+
+ const scanRoot = path.join(root, "scan");
+ await mkdir(scanRoot, { mode: 0o700 });
+ const discovery = path.join(scanRoot, "artifacts", "02_discovery");
+ await mkdir(discovery, { recursive: true });
+ await copyFile(scope, path.join(discovery, "in_scope_files.txt"));
+ const context: ArtifactContext = {
+ root: scanRoot,
+ repoRoot: repo,
+ scanId: "11111111-1111-4111-8111-111111111111",
+ layout: "scan",
+ pluginRoot: path.resolve(import.meta.dirname, "../.."),
+ mode: "diff",
+ targetContract: {
+ diffTarget: {
+ kind: "range",
+ baseRevision: base,
+ headRevision: git("rev-parse", "HEAD"),
+ },
+ },
+ };
+ for (const sourcePath of new Set(["app.py", candidatePath])) {
+ assert.deepEqual(
+ await recordCodexSecurityDiscoveryCandidates(
+ {
+ candidates: [
+ {
+ cwe_ids: ["CWE-20"],
+ locations: [
+ { path: sourcePath, start_line: 1, role: "source" },
+ ],
+ summary: "Synthetic candidate",
+ evidence: "Synthetic evidence",
+ },
+ ],
+ },
+ context,
+ ),
+ { operation: "replace", candidatesRecorded: 1 },
+ );
+ const ledger = await readFile(
+ path.join(discovery, "candidate_ledger.jsonl"),
+ "utf8",
+ );
+ assert.equal(JSON.parse(ledger).locations[0].path, sourcePath);
+ }
+
+ // Candidate JSON remains UTF-8 even when inventory paths contain raw bytes.
+ await writeFile(input, Buffer.from([0xff]));
+ assert.equal(normalizeCandidatesCommand(args), 2);
+ assert.equal(await readFile(output, "utf8"), retained);
+ } finally {
+ await directories.cleanup();
+ }
+ },
+ );
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_compact_artifact_wire.ts b/plugins/codex-security/mcp-app/tests/test_compact_artifact_wire.ts
new file mode 100644
index 000000000..4bb47a9d5
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_compact_artifact_wire.ts
@@ -0,0 +1,220 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import path from "node:path";
+import { createInterface } from "node:readline";
+import { fileURLToPath } from "node:url";
+import { test } from "node:test";
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { encodePosixPath } from "../src/helpers/posix-path.ts";
+import { importSource } from "./import-module.ts";
+import { temporaryDirectory } from "./support/temporary-directories.ts";
+
+const { registerCompactArtifactTools } = await importSource(
+ fileURLToPath(
+ new URL("../src/server/compact-artifact-tools.ts", import.meta.url),
+ ),
+ { loader: { ".md": "text" } },
+);
+const sdkRequire = createRequire(import.meta.resolve("@openai/codex-sdk"));
+const nativeCli = sdkRequire.resolve("@openai/codex/bin/codex.js");
+
+for (const raw of [false, true]) {
+ test(
+ `compact inventory survives native MCP transport with ${raw ? "raw POSIX" : "Unicode"} names`,
+ { skip: raw && process.platform === "win32", timeout: 30_000 },
+ async () => {
+ const root = await temporaryDirectory("compact-artifact-wire-", true);
+ try {
+ const scanDir = path.join(root, "scan");
+ const repository = path.join(root, "repository");
+ const inventoryDir = path.join(scanDir, "artifacts", "02_discovery");
+ await mkdir(inventoryDir, { recursive: true });
+ await mkdir(repository);
+ const inventory = raw
+ ? Buffer.concat([
+ Buffer.from("./name-"),
+ Buffer.from([0xff]),
+ Buffer.from(".ts\n"),
+ ])
+ : Buffer.from("./normal.ts\n./emoji-😀.ts\n");
+ await writeFile(
+ path.join(inventoryDir, "in_scope_files.txt"),
+ inventory,
+ );
+ const callbacks = new Map<
+ string,
+ (input: unknown, context: unknown) => Promise<CallToolResult>
+ >();
+ registerCompactArtifactTools(
+ {
+ registerTool(
+ name: string,
+ _schema: unknown,
+ callback: (
+ input: unknown,
+ context: unknown,
+ ) => Promise<CallToolResult>,
+ ) {
+ callbacks.set(name, callback);
+ },
+ } as unknown as McpServer,
+ {
+ pluginRoot: path.resolve(import.meta.dirname, "../.."),
+ resolveScanRoot: async () => root,
+ runWorkbench: async (args: string[]) => {
+ assert.deepEqual(args, [
+ "get-scan",
+ "--scan-id",
+ "11111111-1111-4111-8111-111111111111",
+ ]);
+ return {
+ scan: {
+ scanId: args[2],
+ scanDir,
+ targetPath: repository,
+ mode: "diff",
+ status: "running",
+ handoffClaimToken: "synthetic-claim",
+ },
+ };
+ },
+ },
+ );
+ const result = await callbacks.get("list_codex_security_review_items")!(
+ {
+ scanId: "11111111-1111-4111-8111-111111111111",
+ handoffClaimToken: "synthetic-claim",
+ },
+ {},
+ );
+ const received = await throughNativeMcp(root, result);
+ const content = received.content[0];
+ assert.equal(content?.type, "text");
+ assert.ok(content && content.type === "text");
+ const parsed = JSON.parse(content.text) as {
+ items: { path: string }[];
+ };
+ if (raw) {
+ assert.equal(received.structuredContent, undefined);
+ assert.deepEqual(
+ encodePosixPath(parsed.items[0]!.path + "\n"),
+ inventory,
+ );
+ } else {
+ assert.deepEqual(parsed, {
+ items: [{ path: "./normal.ts" }, { path: "./emoji-😀.ts" }],
+ });
+ assert.deepEqual(received.structuredContent, parsed);
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+ );
+}
+
+async function throughNativeMcp(
+ root: string,
+ result: CallToolResult,
+): Promise<CallToolResult> {
+ const responsePath = path.join(root, "response.json");
+ const serverPath = path.join(root, "mcp-fixture.mjs");
+ const codexHome = path.join(root, "codex-home");
+ await mkdir(codexHome, { mode: 0o700 });
+ await writeFile(responsePath, JSON.stringify(result));
+ await writeFile(
+ serverPath,
+ `
+import { readFileSync } from 'node:fs';
+import { createInterface } from 'node:readline';
+const result = JSON.parse(readFileSync(process.argv[2], 'utf8'));
+createInterface({input: process.stdin}).on('line', line => {
+ const request = JSON.parse(line);
+ if (request.id === undefined) return;
+ const value = request.method === 'initialize'
+ ? {protocolVersion: request.params.protocolVersion, capabilities: {tools: {}}, serverInfo: {name: 'fixture', version: '1'}}
+ : request.method === 'tools/list'
+ ? {tools: [{name: 'inventory', description: 'Synthetic inventory', inputSchema: {type: 'object', properties: {}}}]}
+ : request.method === 'tools/call' ? result : {};
+ process.stdout.write(JSON.stringify({jsonrpc: '2.0', id: request.id, result: value}) + '\\n');
+});
+`,
+ );
+ const child = spawn(
+ process.execPath,
+ [
+ nativeCli,
+ "app-server",
+ "--stdio",
+ "-c",
+ "features.plugins=false",
+ "-c",
+ `mcp_servers.fixture.command=${JSON.stringify(process.execPath)}`,
+ "-c",
+ `mcp_servers.fixture.args=${JSON.stringify([serverPath, responsePath])}`,
+ "-c",
+ "mcp_servers.fixture.tool_timeout_sec=3",
+ ],
+ {
+ cwd: root,
+ env: { ...process.env, CODEX_HOME: codexHome },
+ windowsHide: true,
+ },
+ );
+ const closed = once(child, "close");
+ const lines = createInterface({ input: child.stdout });
+ let stderr = "";
+ child.stderr.setEncoding("utf8").on("data", (chunk) => {
+ stderr += chunk;
+ });
+ const iterator = lines[Symbol.asyncIterator]();
+ const timer = setTimeout(() => child.kill("SIGKILL"), 20_000);
+ let id = 0;
+ const request = async (method: string, params: unknown) => {
+ const identifier = ++id;
+ child.stdin.write(
+ JSON.stringify({ id: identifier, method, params }) + "\n",
+ );
+ while (true) {
+ const line = await iterator.next();
+ if (line.done) throw new Error(`Native MCP transport closed: ${stderr}`);
+ const response = JSON.parse(line.value);
+ if (response.id !== identifier) continue;
+ if (response.error) throw new Error(JSON.stringify(response.error));
+ return response.result;
+ }
+ };
+ try {
+ await request("initialize", {
+ clientInfo: { name: "compact-artifact-fixture", version: "1" },
+ capabilities: { experimentalApi: true },
+ });
+ child.stdin.write(JSON.stringify({ method: "initialized" }) + "\n");
+ const started = await request("thread/start", {
+ cwd: root,
+ ephemeral: true,
+ approvalPolicy: "never",
+ sandbox: "read-only",
+ });
+ return await request("mcpServer/tool/call", {
+ threadId: started.thread.id,
+ server: "fixture",
+ tool: "inventory",
+ arguments: {},
+ });
+ } finally {
+ clearTimeout(timer);
+ lines.close();
+ child.kill();
+ const force = setTimeout(() => child.kill("SIGKILL"), 3_000);
+ try {
+ await closed;
+ } finally {
+ clearTimeout(force);
+ }
+ }
+}
diff --git a/plugins/codex-security/scripts/generate_in_scope_files.py b/plugins/codex-security/scripts/generate_in_scope_files.py
index 17bd2fb41..b3ebd959a 100644
--- a/plugins/codex-security/scripts/generate_in_scope_files.py
+++ b/plugins/codex-security/scripts/generate_in_scope_files.py
@@ -196,12 +196,7 @@ def generate_diff_in_scope_files(
continue
elif path.is_symlink() or not path.is_file() or is_binary_file(path):
continue
- try:
- relative_path = relative.as_posix().encode("utf-8")
- except UnicodeEncodeError as error:
- raise InventoryError(
- "Git changes contain a path that cannot be encoded as UTF-8 for the file inventory"
- ) from error
+ relative_path = os.fsencode(relative.as_posix())
if b"\n" in relative_path or b"\r" in relative_path:
raise InventoryError(
"Git changes contain a path that cannot fit in the file inventory"
diff --git a/plugins/codex-security/scripts/workbench_source_excerpt.py b/plugins/codex-security/scripts/workbench_source_excerpt.py
index 87d4199c8..1cb80175d 100644
--- a/plugins/codex-security/scripts/workbench_source_excerpt.py
+++ b/plugins/codex-security/scripts/workbench_source_excerpt.py
@@ -69,6 +69,11 @@ def scanned_source_text(scan: sqlite3.Row, target: Path, path: str) -> str | Non
if revision == "unversioned":
return None
snapshot_digest = scan["target_snapshot_digest"]
+ if (
+ scan["diff_target_kind"] == "working_tree"
+ and scan["diff_content_digest"] != clean_worktree_content_digest()
+ ):
+ return None
if snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():
return None
if (
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index b1c962ca9..919f1e674 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -701,7 +701,10 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
junctions[identity] = path
continue
paths.append(path)
- if not stat.S_ISDIR(metadata.st_mode):
+ if (
+ not stat.S_ISDIR(metadata.st_mode)
+ or getattr(metadata, "st_reparse_tag", 0) & 0x20000000
+ ):
continue
nested_repository_root = git_output(path, "rev-parse", "--show-toplevel")
if nested_repository_root is not None and Path(nested_repository_root).samefile(path):
@@ -820,7 +823,10 @@ def source_directory_snapshot_paths(
def directory_content_digest(
- target: Path, *, excluded: tuple[Path, ...] = (), include_ignored: bool = False
+ target: Path,
+ *,
+ excluded: tuple[Path, ...] = (),
+ include_ignored: bool = False,
) -> str:
def raise_walk_error(error: OSError) -> None:
raise error
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
index 097909ea8..1c2d954d3 100644
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py
@@ -794,29 +794,20 @@ def test_diff_inventory_includes_type_changed_files(
@pytest.mark.skipif(os.name == "nt", reason="Windows paths cannot retain arbitrary non-UTF-8 bytes")
-def test_diff_inventory_rejects_non_utf8_path_and_preserves_output(tmp_path: Path) -> None:
+def test_diff_inventory_preserves_non_utf8_path_bytes(tmp_path: Path) -> None:
repository = make_repository(tmp_path)
git(repository, "add", ".")
git(repository, "commit", "-qm", "base")
base = git(repository, "rev-parse", "HEAD")
- blob = git(repository, "rev-parse", f"{base}:app/routes.py")
- subprocess.run(
- ["git", "-C", str(repository), "update-index", "-z", "--index-info"],
- input=b"100644 " + blob.encode("ascii") + b"\ttests/caf\xe9.py\0",
- check=True,
- capture_output=True,
- )
- head = git(
- repository, "commit-tree", git(repository, "write-tree"), "-p", base, "-m", "Byte path"
- )
+ write_file(repository, os.fsdecode(b"app/caf\xe9.py"), b"changed = True\n")
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "Add byte path")
output = tmp_path / "in_scope_files.txt"
- output.write_text("previous.py\n", encoding="utf-8")
result = run_inventory(
- repository, ".", output, arguments=["--diff-base", base, "--diff-head", head]
+ repository, ".", output, arguments=["--diff-base", base, "--diff-head", "HEAD"]
)
- assert result.returncode == 2
- assert "cannot be encoded as UTF-8 for the file inventory" in result.stderr
- assert output.read_text(encoding="utf-8") == "previous.py\n"
+ assert result.returncode == 0, result.stderr
+ assert output.read_bytes() == b"app/caf\xe9.py\n"
def test_local_diff_inventory_skips_unreadable_files(
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index 009ca1e5c..afc86acfb 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -847,3 +847,32 @@ def test_make_rank_input_decodes_bom_marked_utf16_source(tmp_path: Path, mode: s
run_repo_cli(command, repo, output, *arguments)
assert {row["path"]: row["preview"] for row in read_jsonl(output)} == expected
+
+
+def test_revision_preview_uses_the_same_read_window_as_local_files(tmp_path: Path) -> None:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ initialize_repo(repo)
+ source = repo / "example.py"
+ source.write_text("def first():\n pass\n")
+ git(repo, "add", ".")
+ git(repo, "commit", "-qm", "base")
+ base = git(repo, "rev-parse", "HEAD")
+ source.write_text(
+ "def first():\n pass\n#" + "x" * 80_000 + "\ndef outside_window():\n pass\n"
+ )
+ git(repo, "add", ".")
+ git(repo, "commit", "-qm", "long source")
+ output = tmp_path / "preview.jsonl"
+ run_repo_cli(
+ "make-diff-rank-input",
+ repo,
+ output,
+ "--base",
+ base,
+ "--head",
+ "HEAD",
+ )
+ rows = read_jsonl(output)
+ assert "first" in rows[0]["preview"]
+ assert "outside_window" not in rows[0]["preview"]
diff --git a/plugins/codex-security/tests/test_git_inventory_regressions.py b/plugins/codex-security/tests/test_git_inventory_regressions.py
index b86ed4089..79ea8914d 100644
--- a/plugins/codex-security/tests/test_git_inventory_regressions.py
+++ b/plugins/codex-security/tests/test_git_inventory_regressions.py
@@ -175,6 +175,36 @@ def test_dirty_submodule_warning_preserves_the_changed_target_detail(tmp_path: P
assert "results were saved" in warning
+def test_committed_binary_detection_agrees_beyond_preview_window(tmp_path: Path) -> None:
+ target = tmp_path / "target"
+ initialize_git_repository(target)
+ base = git(target, "rev-parse", "HEAD").decode()
+ (target / "payload.php").write_bytes(b"<?php\n" + b" " * (70 * 1024) + b"\0payload")
+ (target / "visible.py").write_text("value = 1\n")
+ git(target, "add", ".")
+ git(target, "commit", "-qm", "Add source and binary fixtures")
+ inventory_path = tmp_path / "inventory.txt"
+ rank_path = tmp_path / "rank.jsonl"
+ load_script("generate_in_scope_files").generate_diff_in_scope_files(
+ target, base, "HEAD", "revisions", inventory_path
+ )
+ load_script("generate_rank_input").make_diff_rank_input(
+ argparse.Namespace(
+ repo=str(target),
+ base=base,
+ head="HEAD",
+ mode="revisions",
+ out=str(rank_path),
+ area="fixture",
+ preview_bytes=1024,
+ )
+ )
+ assert inventory_path.read_text().splitlines() == ["visible.py"]
+ assert [json.loads(line)["path"] for line in rank_path.read_text().splitlines()] == [
+ "visible.py"
+ ]
+
+
def test_disabled_git_source_snapshot_does_not_inspect_refs(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts
index 45d6a9753..f32a3bf11 100644
--- a/sdk/typescript/tests-ts/multiscan.test.ts
+++ b/sdk/typescript/tests-ts/multiscan.test.ts
@@ -3474,11 +3474,21 @@ describe("multiscan", () => {
});
});
-test.each(["sha1", "sha256"])(
- "bulk checkout preserves %s object format despite Git defaults",
- async (objectFormat) => {
- const name = `bulk checkout preserves ${objectFormat} object format despite Git defaults`;
- if (runTestInSubprocess(fileURLToPath(import.meta.url), name)) return;
+test.each([
+ ["sha1", "sha1"],
+ ["sha1", "sha256"],
+ ["sha256", "sha1"],
+ ["sha256", "sha256"],
+])(
+ "bulk checkout preserves %s object format with a %s default",
+ async (objectFormat, defaultFormat) => {
+ if (
+ runTestInSubprocess(
+ fileURLToPath(import.meta.url),
+ `bulk checkout preserves ${objectFormat} object format with a ${defaultFormat} default`,
+ )
+ )
+ return;
const paths = await fixture();
const repo = await repository(paths.root, "source", objectFormat);
await writeFile(
@@ -3486,24 +3496,75 @@ test.each(["sha1", "sha256"])(
`id,repository,revision\nfixture,${repo.path},${repo.revision}\n`,
);
const run = mock(async (checkout: string, scanOptions = {}) => {
+ expect(process.env["GIT_DEFAULT_HASH"]).toBe(defaultFormat);
expect(git(checkout, "rev-parse", "--show-object-format")).toBe(
objectFormat,
);
+ expect(git(checkout, "rev-parse", "HEAD")).toBe(repo.revision);
return completeRun(checkout, scanOptions);
});
const previous = process.env["GIT_DEFAULT_HASH"];
- process.env["GIT_DEFAULT_HASH"] =
- objectFormat === "sha1" ? "sha256" : "sha1";
+ process.env["GIT_DEFAULT_HASH"] = defaultFormat;
try {
const result = await runMultiscan(options(paths, client(run)));
expect(result.failed).toBe(0);
expect(run).toHaveBeenCalledTimes(1);
- expect(process.env["GIT_DEFAULT_HASH"]).toBe(
- objectFormat === "sha1" ? "sha256" : "sha1",
- );
+ expect(process.env["GIT_DEFAULT_HASH"]).toBe(defaultFormat);
} finally {
if (previous === undefined) delete process.env["GIT_DEFAULT_HASH"];
else process.env["GIT_DEFAULT_HASH"] = previous;
}
},
);
+
+testPosix(
+ "bulk SHA-1 checkout works without Git object-format options",
+ async () => {
+ if (
+ runTestInSubprocess(
+ fileURLToPath(import.meta.url),
+ "bulk SHA-1 checkout works without Git object-format options",
+ )
+ )
+ return;
+ const paths = await fixture();
+ const source = await repository(paths.root, "source");
+ await writeFile(
+ paths.input,
+ `id,repository,revision\nfixture,${source.path},${source.revision}\n`,
+ );
+ const command = await resolveTrustedExecutable(
+ "git",
+ process.env,
+ process.cwd(),
+ );
+ expect(command).not.toBeNull();
+ const bin = await temporaryDirectory();
+ await writeFile(
+ join(bin, "git"),
+ `#!/bin/sh
+for argument do
+ case "$argument" in
+ --object-format=*) echo 'error: unknown option object-format' >&2; exit 129 ;;
+ esac
+done
+exec '${command!.executable.replaceAll("'", "'\\''")}' "$@"
+`,
+ { mode: 0o700 },
+ );
+ const previousPath = process.env["PATH"];
+ process.env["PATH"] = `${bin}:${previousPath ?? ""}`;
+ try {
+ const run = mock(async (checkout: string, scanOptions = {}) => {
+ expect(git(checkout, "rev-parse", "HEAD")).toBe(source.revision);
+ return completeRun(checkout, scanOptions);
+ });
+ const result = await runMultiscan(options(paths, client(run)));
+ expect(result.failed).toBe(0);
+ expect(run).toHaveBeenCalledTimes(1);
+ } finally {
+ if (previousPath === undefined) delete process.env["PATH"];
+ else process.env["PATH"] = previousPath;
+ }
+ },
+);
diff --git a/sdk/typescript/tests-ts/plugin-report-limits.test.ts b/sdk/typescript/tests-ts/plugin-report-limits.test.ts
index 7f1b044d5..4f2617ea9 100644
--- a/sdk/typescript/tests-ts/plugin-report-limits.test.ts
+++ b/sdk/typescript/tests-ts/plugin-report-limits.test.ts
@@ -50,6 +50,7 @@ describe("bundled scan report and source limits", () => {
"diagnostics = {'rootCause': {'summary': 'Missing authorization check.'}, 'validation': {'summary': 'An untrusted request reaches the protected resource.'}, 'attackPath': {'narrative': 'The request bypasses the authorization boundary.'}, 'codeEvidence': [{'id': 'evidence', 'label': 'Missing check', 'path': 'example.py', 'startLine': 1, 'code': 'return resource', 'explanation': 'No authorization check runs.'}], 'evidence': 'The protected resource was exposed.', 'evidenceExcerpt': 'return resource'}",
"details = {'remediationTests': [f'test-{index}' for index in range(40)], 'preventiveControls': [f'control-{index}' for index in range(40)]}",
"large = {**diagnostics, 'preventiveControls': ['x' * 900 for _ in range(20)], 'remediationTests': ['Verify authorization.'], 'writeup': {'reportPath': 'findings/example/example.md'}, 'provenance': {'source': 'scan'}, 'severity': {'level': 'high', 'rationale': 'Verified impact'}, 'status': 'open', 'taxonomy': {'category': 'injection', 'cwe': ['CWE-79']}}",
+ "oversized_severity = {**large, 'severity': {'level': 'high', 'rationale': 'x' * 18850}}",
"code_evidence = [{'id': f'evidence-{index}', 'label': 'example', 'path': 'example.py', 'startLine': 1, 'code': 'c' * 1500, 'explanation': 'e' * 1500} for index in range(4)]",
"rich = {'rootCause': {'summary': 'r' * 2000}, 'validation': {'summary': 'v' * 3000}, 'attackPath': {'narrative': 'a' * 4000}, 'codeEvidence': code_evidence, 'evidenceExcerpt': 'e' * 8000, 'identity': {'anchor': 'finding'}, 'preventiveControls': ['Centralize authorization.'], 'remediationTests': ['Verify authorization.']}",
"boundary = {**diagnostics, 'remediationTests': ['x'] * 4000, 'preventiveControls': ['Keep authorization centralized.']}",
@@ -59,7 +60,7 @@ describe("bundled scan report and source limits", () => {
"oversized_metadata = {**diagnostics, 'confidence': {'level': 'high', 'rationale': 'x' * 17000}, 'remediationTests': ['Verify authorization.'], 'preventiveControls': ['Centralize authorization.']}",
"oversized_guidance = {'rootCause': {'summary': 'root'}, 'validation': {'summary': 'validation'}, 'attackPath': {'narrative': 'attack'}, 'codeEvidence': [{'id': 'evidence', 'label': 'evidence', 'path': 'example.py', 'startLine': 1, 'code': 'x', 'explanation': 'evidence'}], 'evidence': 'legacy', 'evidenceExcerpt': 'excerpt', 'remediationTests': ['x' * 7800], 'preventiveControls': ['y' * 7930]}",
"nested_boundary = {'remediationTests': ['x'] * 3937, 'rootCause': {'summary': 'r' * 178, 'detail': {'x': {'y': 'z'}}}}",
- "projections = {key: bounded_finding_details(value) for key, value in {'details': details, 'large': large, 'rich': rich, 'boundary': boundary, 'unicodeBoundary': unicode_boundary, 'emptyControls': empty_controls, 'emptyTests': empty_tests, 'oversizedMetadata': oversized_metadata, 'oversizedGuidance': oversized_guidance, 'nestedBoundary': nested_boundary}.items()}",
+ "projections = {key: bounded_finding_details(value) for key, value in {'details': details, 'large': large, 'rich': rich, 'boundary': boundary, 'unicodeBoundary': unicode_boundary, 'emptyControls': empty_controls, 'emptyTests': empty_tests, 'oversizedMetadata': oversized_metadata, 'oversizedSeverity': oversized_severity, 'oversizedGuidance': oversized_guidance, 'nestedBoundary': nested_boundary}.items()}",
"print(json.dumps({'projections': projections, 'bytes': {key: len(json.dumps(value, separators=(',', ':')).encode()) for key, value in projections.items()}}))",
].join("\n");
const result = Bun.spawnSync(
@@ -88,6 +89,7 @@ describe("bundled scan report and source limits", () => {
remediationTests: string[];
preventiveControls: string[];
};
+ oversizedSeverity: Record<string, unknown>;
oversizedMetadata: {
remediationTests: string[];
preventiveControls: string[];
@@ -121,6 +123,9 @@ describe("bundled scan report and source limits", () => {
status: "open",
taxonomy: { category: "injection", cwe: ["CWE-79"] },
});
+ expect(projections.oversizedSeverity).toMatchObject({
+ writeup: { reportPath: "findings/example/example.md" },
+ });
expect(projections.rich).toMatchObject({
identity: { anchor: "finding" },
preventiveControls: ["Centralize authorization."],
@@ -134,6 +139,7 @@ describe("bundled scan report and source limits", () => {
projections.emptyControls,
projections.emptyTests,
projections.oversizedMetadata,
+ projections.oversizedSeverity,
projections.oversizedGuidance,
]) {
expect(finding).toMatchObject({