OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(targets): preserve Git checkout formats and raw POSIX inventory paths (#1288)
Michael D'Angeloe1adad147a88
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.ymlindex 669acae8b..c96b833aa 100644--- a/.github/workflows/node-ci.yml+++ b/.github/workflows/node-ci.yml@@ -421,7 +421,7 @@ jobs:python -m pytest plugins/codex-security/tests -m "$PYTEST_MARKERS" -q -n 4 --dist worksteal --max-worker-restart 0 --durations=30 --junitxml=reports/python.xml- name: Require native platform coverageif: runner.os != 'Linux'- run: python -c "import xml.etree.ElementTree as ET; assert not ET.parse('reports/python.xml').findall('.//testcase/skipped'), 'Native platform tests must run without skips'"+ run: python -c "import pathlib, xml.etree.ElementTree as ET; assert not any(ET.parse(report).findall('.//testcase/skipped') for report in pathlib.Path('reports').glob('python*.xml')), 'Native platform tests must run without skips'"- name: Upload Python test reportsif: always()continue-on-error: true@@ -429,7 +429,7 @@ jobs:with:name: python-test-reports-${{ matrix.os }}-${{ matrix.python }}overwrite: true- path: reports/python.xml+ path: reports/python*.xmlif-no-files-found: warnretention-days: 14- name: Set up Node.js for triage evalsdiff --git a/plugins/codex-security/mcp-app/src/artifact-discovery.ts b/plugins/codex-security/mcp-app/src/artifact-discovery.tsindex d04a99668..0d736551c 100644--- a/plugins/codex-security/mcp-app/src/artifact-discovery.ts+++ b/plugins/codex-security/mcp-app/src/artifact-discovery.ts@@ -3,6 +3,7 @@ import { createRequire } from "node:module";import { join } from "node:path";import { Worker } from "node:worker_threads";import { decodeUtf8 } from "./helpers/utf8.js";+import { decodePosixBytes } from "./helpers/posix-path.js";import { resolvePythonCommand, runPythonWithInput } from "./python_command.js";import {stableJson,@@ -161,7 +162,12 @@ async function diffCandidateSources(}const nativePath = (value: string) =>process.platform === "win32" ? value.replaceAll("\\", "/") : value;- const paths = decodeUtf8(await readFile(inventory))+ const contents = await readFile(inventory);+ const paths = (+ process.platform === "win32"+ ? decodeUtf8(contents)+ : decodePosixBytes(contents)+ ).split(/\r?\n/u).filter(Boolean).map(nativePath);diff --git a/plugins/codex-security/mcp-app/src/artifact-inventory.ts b/plugins/codex-security/mcp-app/src/artifact-inventory.tsindex 4852f1491..23d92a107 100644--- a/plugins/codex-security/mcp-app/src/artifact-inventory.ts+++ b/plugins/codex-security/mcp-app/src/artifact-inventory.ts@@ -21,6 +21,7 @@ import {resolvePythonCommand,} from "./python_command.js";import { decodeUtf8 } from "./helpers/utf8.js";+import { decodePosixBytes } from "./helpers/posix-path.js";const execFile = promisify(nodeExecFile);const documents = [commonSchema, reviewItemsSchema] as SchemaDocument[];@@ -159,7 +160,10 @@ async function readReviewItems({ cause: error },);});- const source = decodeUtf8(contents);+ const source =+ process.platform === "win32"+ ? decodeUtf8(contents)+ : decodePosixBytes(contents);const rows: ReviewItem[] = [];for (const [index, line] of source.split(/\r?\n/u).entries()) {diff --git a/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.ts b/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.tsindex 731458293..6b784beb3 100644--- a/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.ts+++ b/plugins/codex-security/mcp-app/src/helpers/normalize-candidates.ts@@ -148,9 +148,67 @@ function readScope(allowMissing: boolean,sources?: ReadonlyMap<string, CandidateSource>,): Set<string> {- const contents = decodeUtf8(readFile(path));+ const bytes = readFile(path);+ const contents = windows ? decodeUtf8(bytes) : decodePosixBytes(bytes);+ const lines = contents.split("\n");+ const listedRows = new Set(lines);+ const isScopeFile = (value: string): boolean => {+ try {+ relativeFile(value, root, sources);+ return true;+ } catch {+ return false;+ }+ };+ const carriageRows = new Map<string, [boolean, boolean]>();+ if (!windows) {+ for (const line of lines) {+ if (line.endsWith("\r") && line !== "\r") {+ carriageRows.set(line, [+ isScopeFile(line),+ isScopeFile(line.slice(0, -1)),+ ]);+ }+ }+ }+ const crlfEvidence = lines.slice(0, -1).some((line) => {+ if (line === "\r") return true;+ const paths = carriageRows.get(line);+ return paths !== undefined && paths[1] && !paths[0];+ });+ const literalEvidence = [...carriageRows.values()].some(+ ([literal, stripped]) => literal && !stripped,+ );const scope = new Set<string>();- for (const [index, line] of contents.split(/\r?\n/u).entries()) {+ for (const [index, row] of lines.entries()) {+ let line = row;+ const finalLiteral = index === lines.length - 1;+ if (!finalLiteral && (windows || line === "\r")) {+ if (line.endsWith("\r")) line = line.slice(0, -1);+ } else if (!finalLiteral && line.endsWith("\r")) {+ const [literal, stripped] = carriageRows.get(line)!;+ const trimmed = line.slice(0, -1);+ if (allowMissing && literal && !stripped && !listedRows.has(trimmed)) {+ throw new Error(+ `in-scope file row ${index + 1}: ambiguous carriage-return paths`,+ );+ }+ if (stripped && !literal) {+ line = trimmed;+ } else if (stripped && literal) {+ if (!listedRows.has(trimmed)) {+ if (crlfEvidence && !literalEvidence) {+ line = trimmed;+ } else if (!literalEvidence || crlfEvidence) {+ throw new Error(+ `in-scope file row ${index + 1}: ambiguous carriage-return paths`,+ );+ }+ }+ } else if (!literal && crlfEvidence) {+ line = trimmed;+ }+ }if (line === "") continue;try {scope.add(relativeFile(line, root, sources)[0]);diff --git a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.tsindex 6a03e7687..ff9ab1499 100644--- a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts+++ b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts@@ -336,9 +336,20 @@ function registerCompactTool<Input>(},async (input, requestContext) => {const value = await registration.handler(input, requestContext);+ let structured = true;+ const text = JSON.stringify(value, (key, item: unknown) => {+ if (+ !key.isWellFormed() ||+ (typeof item === "string" && !item.isWellFormed())+ )+ structured = false;+ return item;+ });+ // Codex's native JSON decoder rejects lone surrogates; the JSON text retains+ // their exact escaped representation, including raw POSIX filename bytes.return {- content: [{ type: "text" as const, text: JSON.stringify(value) }],- structuredContent: value as JsonRecord,+ content: [{ type: "text" as const, text }],+ ...(structured ? { structuredContent: value as JsonRecord } : {}),};},);diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts b/plugins/codex-security/mcp-app/tests/test_artifact_discovery.tsindex ab6802d3c..8a4e6321b 100644--- a/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts+++ b/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts@@ -407,7 +407,9 @@ async function verifyNormalizerFailuresPreserveOutput(await writeFile(inventory, Buffer.from([0xff]));await assert.rejects(recordCodexSecurityDiscoveryCandidates({ candidates: [] }, context),- /UTF-8|encoded data/,+ process.platform === "win32"+ ? /UTF-8|encoded data/+ : /in-scope file row 1:.*ENOENT/,);assert.equal(await readFile(destination, "utf8"), original);await writeFile(inventory, scope);diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.tsindex 86082c9f3..cd746f175 100644--- a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts+++ b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts@@ -137,14 +137,18 @@ async function testNonUtf8InventoryPaths() {]);await writeFile(rawPath, "raw path\n");await fixture.writeRepositoryFile("name-�.ts", "Unicode path\n");- await assert.rejects(- inventory.prepareCodexSecurityReviewItems(fixture.scan),- /not valid for encoding utf-8/,+ assert.deepEqual(+ await inventory.prepareCodexSecurityReviewItems(fixture.scan),+ {+ reviewItemsTotal: 2,+ },);assert.ok((await readFile(fixture.scanInventory)).includes(0xff));- await assert.rejects(- inventory.listCodexSecurityReviewItems(fixture.scan),- /not valid for encoding utf-8/,+ assert.deepEqual(+ (await inventory.listCodexSecurityReviewItems(fixture.scan)).items+ .map((item: { path: string }) => item.path)+ .sort(),+ ["./name-\udcff.ts", "./name-�.ts"],);await unlink(rawPath);assert.deepEqual(diff --git a/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts b/plugins/codex-security/mcp-app/tests/test_candidate_inventory.tsnew file mode 100644index 000000000..a26d8363b--- /dev/null+++ b/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts@@ -0,0 +1,171 @@+import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import { copyFile, mkdir, readFile, unlink, writeFile } from "node:fs/promises";+import path from "node:path";+import { test } from "node:test";+import type { ArtifactContext } from "../src/artifact-io.js";+import { importSource } from "./import-module.ts";+import { createTemporaryDirectories } from "./support/temporary-directories.ts";++const { normalizeCandidatesCommand } = await importSource(+ new URL("../src/helpers/normalize-candidates.ts", import.meta.url).pathname,+);+const { recordCodexSecurityDiscoveryCandidates } = await importSource(+ path.join(import.meta.dirname, "../src/artifact-discovery.ts"),+ {+ define: {+ "import.meta.url": JSON.stringify(+ new URL(+ "../../../../sdk/typescript/_bundled_plugin/mcp/server.mjs",+ import.meta.url,+ ).href,+ ),+ },+ },+);+const inventoryScript = path.join(+ import.meta.dirname,+ "../../scripts/generate_in_scope_files.py",+);++for (const deleted of [false, true]) {+ test(+ `normalizes a generated inventory with a ${deleted ? "deleted" : "present"} raw-byte path`,+ { skip: process.platform !== "linux" },+ async () => {+ const directories = createTemporaryDirectories(true);+ const root = await directories.create("candidate-inventory-");+ try {+ const repo = path.join(root, "repository");+ await mkdir(repo);+ const rawName = Buffer.from([0x63, 0x61, 0x66, 0xe9, 0x2e, 0x70, 0x79]);+ const rawPath = Buffer.concat([Buffer.from(`${repo}/`), rawName]);+ const normalPath = path.join(repo, "app.py");+ await writeFile(rawPath, "first\n");+ await writeFile(normalPath, "first\n");+ const git = (...args: string[]) =>+ execFileSync(+ "git",+ [+ "-c",+ "user.name=Fixture",+ "-c",+ "[email protected]",+ ...args,+ ],+ { cwd: repo, encoding: "utf8" },+ ).trim();+ git("init", "-q");+ git("add", ".");+ git("commit", "-qm", "initial");+ const base = git("rev-parse", "HEAD");+ if (deleted) await unlink(rawPath);+ else await writeFile(rawPath, "changed\n");+ await writeFile(normalPath, "changed\n");+ git("add", ".");+ git("commit", "-qm", "change");++ const scope = path.join(root, "in-scope.txt");+ execFileSync(process.env.PYTHON ?? "python3", [+ inventoryScript,+ "--repo",+ repo,+ "--scope",+ ".",+ "--out",+ scope,+ "--diff-base",+ base,+ "--diff-head",+ "HEAD",+ ]);+ assert.deepEqual(+ await readFile(scope),+ Buffer.concat([Buffer.from("app.py\n"), rawName, Buffer.from("\n")]),+ );+ const input = path.join(root, "candidates.jsonl");+ const output = path.join(root, "combined.jsonl");+ const candidatePath = deleted ? "app.py" : "caf\udce9.py";+ await writeFile(+ input,+ JSON.stringify({+ cwe_ids: ["CWE-20"],+ locations: [{ path: candidatePath, start_line: 1, role: "source" }],+ summary: "Synthetic candidate",+ evidence: "Synthetic evidence",+ }) + "\n",+ );+ const args = [+ "--input",+ input,+ "--out",+ output,+ "--repo-root",+ repo,+ "--in-scope-files",+ scope,+ ];+ if (deleted) {+ assert.equal(normalizeCandidatesCommand(args), 2);+ args.push("--allow-missing-in-scope");+ }+ assert.equal(normalizeCandidatesCommand(args), 0);+ const retained = await readFile(output, "utf8");+ assert.equal(JSON.parse(retained).locations[0].path, candidatePath);++ const scanRoot = path.join(root, "scan");+ await mkdir(scanRoot, { mode: 0o700 });+ const discovery = path.join(scanRoot, "artifacts", "02_discovery");+ await mkdir(discovery, { recursive: true });+ await copyFile(scope, path.join(discovery, "in_scope_files.txt"));+ const context: ArtifactContext = {+ root: scanRoot,+ repoRoot: repo,+ scanId: "11111111-1111-4111-8111-111111111111",+ layout: "scan",+ pluginRoot: path.resolve(import.meta.dirname, "../.."),+ mode: "diff",+ targetContract: {+ diffTarget: {+ kind: "range",+ baseRevision: base,+ headRevision: git("rev-parse", "HEAD"),+ },+ },+ };+ for (const sourcePath of new Set(["app.py", candidatePath])) {+ assert.deepEqual(+ await recordCodexSecurityDiscoveryCandidates(+ {+ candidates: [+ {+ cwe_ids: ["CWE-20"],+ locations: [+ { path: sourcePath, start_line: 1, role: "source" },+ ],+ summary: "Synthetic candidate",+ evidence: "Synthetic evidence",+ },+ ],+ },+ context,+ ),+ { operation: "replace", candidatesRecorded: 1 },+ );+ const ledger = await readFile(+ path.join(discovery, "candidate_ledger.jsonl"),+ "utf8",+ );+ assert.equal(JSON.parse(ledger).locations[0].path, sourcePath);+ }++ // Candidate JSON remains UTF-8 even when inventory paths contain raw bytes.+ await writeFile(input, Buffer.from([0xff]));+ assert.equal(normalizeCandidatesCommand(args), 2);+ assert.equal(await readFile(output, "utf8"), retained);+ } finally {+ await directories.cleanup();+ }+ },+ );+}diff --git a/plugins/codex-security/mcp-app/tests/test_compact_artifact_wire.ts b/plugins/codex-security/mcp-app/tests/test_compact_artifact_wire.tsnew file mode 100644index 000000000..4bb47a9d5--- /dev/null+++ b/plugins/codex-security/mcp-app/tests/test_compact_artifact_wire.ts@@ -0,0 +1,220 @@+import assert from "node:assert/strict";+import { spawn } from "node:child_process";+import { once } from "node:events";+import { mkdir, rm, writeFile } from "node:fs/promises";+import { createRequire } from "node:module";+import path from "node:path";+import { createInterface } from "node:readline";+import { fileURLToPath } from "node:url";+import { test } from "node:test";+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";+import { encodePosixPath } from "../src/helpers/posix-path.ts";+import { importSource } from "./import-module.ts";+import { temporaryDirectory } from "./support/temporary-directories.ts";++const { registerCompactArtifactTools } = await importSource(+ fileURLToPath(+ new URL("../src/server/compact-artifact-tools.ts", import.meta.url),+ ),+ { loader: { ".md": "text" } },+);+const sdkRequire = createRequire(import.meta.resolve("@openai/codex-sdk"));+const nativeCli = sdkRequire.resolve("@openai/codex/bin/codex.js");++for (const raw of [false, true]) {+ test(+ `compact inventory survives native MCP transport with ${raw ? "raw POSIX" : "Unicode"} names`,+ { skip: raw && process.platform === "win32", timeout: 30_000 },+ async () => {+ const root = await temporaryDirectory("compact-artifact-wire-", true);+ try {+ const scanDir = path.join(root, "scan");+ const repository = path.join(root, "repository");+ const inventoryDir = path.join(scanDir, "artifacts", "02_discovery");+ await mkdir(inventoryDir, { recursive: true });+ await mkdir(repository);+ const inventory = raw+ ? Buffer.concat([+ Buffer.from("./name-"),+ Buffer.from([0xff]),+ Buffer.from(".ts\n"),+ ])+ : Buffer.from("./normal.ts\n./emoji-😀.ts\n");+ await writeFile(+ path.join(inventoryDir, "in_scope_files.txt"),+ inventory,+ );+ const callbacks = new Map<+ string,+ (input: unknown, context: unknown) => Promise<CallToolResult>+ >();+ registerCompactArtifactTools(+ {+ registerTool(+ name: string,+ _schema: unknown,+ callback: (+ input: unknown,+ context: unknown,+ ) => Promise<CallToolResult>,+ ) {+ callbacks.set(name, callback);+ },+ } as unknown as McpServer,+ {+ pluginRoot: path.resolve(import.meta.dirname, "../.."),+ resolveScanRoot: async () => root,+ runWorkbench: async (args: string[]) => {+ assert.deepEqual(args, [+ "get-scan",+ "--scan-id",+ "11111111-1111-4111-8111-111111111111",+ ]);+ return {+ scan: {+ scanId: args[2],+ scanDir,+ targetPath: repository,+ mode: "diff",+ status: "running",+ handoffClaimToken: "synthetic-claim",+ },+ };+ },+ },+ );+ const result = await callbacks.get("list_codex_security_review_items")!(+ {+ scanId: "11111111-1111-4111-8111-111111111111",+ handoffClaimToken: "synthetic-claim",+ },+ {},+ );+ const received = await throughNativeMcp(root, result);+ const content = received.content[0];+ assert.equal(content?.type, "text");+ assert.ok(content && content.type === "text");+ const parsed = JSON.parse(content.text) as {+ items: { path: string }[];+ };+ if (raw) {+ assert.equal(received.structuredContent, undefined);+ assert.deepEqual(+ encodePosixPath(parsed.items[0]!.path + "\n"),+ inventory,+ );+ } else {+ assert.deepEqual(parsed, {+ items: [{ path: "./normal.ts" }, { path: "./emoji-😀.ts" }],+ });+ assert.deepEqual(received.structuredContent, parsed);+ }+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ },+ );+}++async function throughNativeMcp(+ root: string,+ result: CallToolResult,+): Promise<CallToolResult> {+ const responsePath = path.join(root, "response.json");+ const serverPath = path.join(root, "mcp-fixture.mjs");+ const codexHome = path.join(root, "codex-home");+ await mkdir(codexHome, { mode: 0o700 });+ await writeFile(responsePath, JSON.stringify(result));+ await writeFile(+ serverPath,+ `+import { readFileSync } from 'node:fs';+import { createInterface } from 'node:readline';+const result = JSON.parse(readFileSync(process.argv[2], 'utf8'));+createInterface({input: process.stdin}).on('line', line => {+ const request = JSON.parse(line);+ if (request.id === undefined) return;+ const value = request.method === 'initialize'+ ? {protocolVersion: request.params.protocolVersion, capabilities: {tools: {}}, serverInfo: {name: 'fixture', version: '1'}}+ : request.method === 'tools/list'+ ? {tools: [{name: 'inventory', description: 'Synthetic inventory', inputSchema: {type: 'object', properties: {}}}]}+ : request.method === 'tools/call' ? result : {};+ process.stdout.write(JSON.stringify({jsonrpc: '2.0', id: request.id, result: value}) + '\\n');+});+`,+ );+ const child = spawn(+ process.execPath,+ [+ nativeCli,+ "app-server",+ "--stdio",+ "-c",+ "features.plugins=false",+ "-c",+ `mcp_servers.fixture.command=${JSON.stringify(process.execPath)}`,+ "-c",+ `mcp_servers.fixture.args=${JSON.stringify([serverPath, responsePath])}`,+ "-c",+ "mcp_servers.fixture.tool_timeout_sec=3",+ ],+ {+ cwd: root,+ env: { ...process.env, CODEX_HOME: codexHome },+ windowsHide: true,+ },+ );+ const closed = once(child, "close");+ const lines = createInterface({ input: child.stdout });+ let stderr = "";+ child.stderr.setEncoding("utf8").on("data", (chunk) => {+ stderr += chunk;+ });+ const iterator = lines[Symbol.asyncIterator]();+ const timer = setTimeout(() => child.kill("SIGKILL"), 20_000);+ let id = 0;+ const request = async (method: string, params: unknown) => {+ const identifier = ++id;+ child.stdin.write(+ JSON.stringify({ id: identifier, method, params }) + "\n",+ );+ while (true) {+ const line = await iterator.next();+ if (line.done) throw new Error(`Native MCP transport closed: ${stderr}`);+ const response = JSON.parse(line.value);+ if (response.id !== identifier) continue;+ if (response.error) throw new Error(JSON.stringify(response.error));+ return response.result;+ }+ };+ try {+ await request("initialize", {+ clientInfo: { name: "compact-artifact-fixture", version: "1" },+ capabilities: { experimentalApi: true },+ });+ child.stdin.write(JSON.stringify({ method: "initialized" }) + "\n");+ const started = await request("thread/start", {+ cwd: root,+ ephemeral: true,+ approvalPolicy: "never",+ sandbox: "read-only",+ });+ return await request("mcpServer/tool/call", {+ threadId: started.thread.id,+ server: "fixture",+ tool: "inventory",+ arguments: {},+ });+ } finally {+ clearTimeout(timer);+ lines.close();+ child.kill();+ const force = setTimeout(() => child.kill("SIGKILL"), 3_000);+ try {+ await closed;+ } finally {+ clearTimeout(force);+ }+ }+}diff --git a/plugins/codex-security/scripts/generate_in_scope_files.py b/plugins/codex-security/scripts/generate_in_scope_files.pyindex 17bd2fb41..b3ebd959a 100644--- a/plugins/codex-security/scripts/generate_in_scope_files.py+++ b/plugins/codex-security/scripts/generate_in_scope_files.py@@ -196,12 +196,7 @@ def generate_diff_in_scope_files(continueelif path.is_symlink() or not path.is_file() or is_binary_file(path):continue- try:- relative_path = relative.as_posix().encode("utf-8")- except UnicodeEncodeError as error:- raise InventoryError(- "Git changes contain a path that cannot be encoded as UTF-8 for the file inventory"- ) from error+ relative_path = os.fsencode(relative.as_posix())if b"\n" in relative_path or b"\r" in relative_path:raise InventoryError("Git changes contain a path that cannot fit in the file inventory"diff --git a/plugins/codex-security/scripts/workbench_source_excerpt.py b/plugins/codex-security/scripts/workbench_source_excerpt.pyindex 87d4199c8..1cb80175d 100644--- a/plugins/codex-security/scripts/workbench_source_excerpt.py+++ b/plugins/codex-security/scripts/workbench_source_excerpt.py@@ -69,6 +69,11 @@ def scanned_source_text(scan: sqlite3.Row, target: Path, path: str) -> str | Nonif revision == "unversioned":return Nonesnapshot_digest = scan["target_snapshot_digest"]+ if (+ scan["diff_target_kind"] == "working_tree"+ and scan["diff_content_digest"] != clean_worktree_content_digest()+ ):+ return Noneif snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():return Noneif (diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.pyindex b1c962ca9..919f1e674 100644--- a/plugins/codex-security/scripts/workbench_target.py+++ b/plugins/codex-security/scripts/workbench_target.py@@ -701,7 +701,10 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:junctions[identity] = pathcontinuepaths.append(path)- if not stat.S_ISDIR(metadata.st_mode):+ if (+ not stat.S_ISDIR(metadata.st_mode)+ or getattr(metadata, "st_reparse_tag", 0) & 0x20000000+ ):continuenested_repository_root = git_output(path, "rev-parse", "--show-toplevel")if nested_repository_root is not None and Path(nested_repository_root).samefile(path):@@ -820,7 +823,10 @@ def source_directory_snapshot_paths(def directory_content_digest(- target: Path, *, excluded: tuple[Path, ...] = (), include_ignored: bool = False+ target: Path,+ *,+ excluded: tuple[Path, ...] = (),+ include_ignored: bool = False,) -> str:def raise_walk_error(error: OSError) -> None:raise errordiff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.pyindex 097909ea8..1c2d954d3 100644--- a/plugins/codex-security/tests/test_generate_in_scope_files.py+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py@@ -794,29 +794,20 @@ def test_diff_inventory_includes_type_changed_files(@pytest.mark.skipif(os.name == "nt", reason="Windows paths cannot retain arbitrary non-UTF-8 bytes")-def test_diff_inventory_rejects_non_utf8_path_and_preserves_output(tmp_path: Path) -> None:+def test_diff_inventory_preserves_non_utf8_path_bytes(tmp_path: Path) -> None:repository = make_repository(tmp_path)git(repository, "add", ".")git(repository, "commit", "-qm", "base")base = git(repository, "rev-parse", "HEAD")- blob = git(repository, "rev-parse", f"{base}:app/routes.py")- subprocess.run(- ["git", "-C", str(repository), "update-index", "-z", "--index-info"],- input=b"100644 " + blob.encode("ascii") + b"\ttests/caf\xe9.py\0",- check=True,- capture_output=True,- )- head = git(- repository, "commit-tree", git(repository, "write-tree"), "-p", base, "-m", "Byte path"- )+ write_file(repository, os.fsdecode(b"app/caf\xe9.py"), b"changed = True\n")+ git(repository, "add", ".")+ git(repository, "commit", "-qm", "Add byte path")output = tmp_path / "in_scope_files.txt"- output.write_text("previous.py\n", encoding="utf-8")result = run_inventory(- repository, ".", output, arguments=["--diff-base", base, "--diff-head", head]+ repository, ".", output, arguments=["--diff-base", base, "--diff-head", "HEAD"])- assert result.returncode == 2- assert "cannot be encoded as UTF-8 for the file inventory" in result.stderr- assert output.read_text(encoding="utf-8") == "previous.py\n"+ assert result.returncode == 0, result.stderr+ assert output.read_bytes() == b"app/caf\xe9.py\n"def test_local_diff_inventory_skips_unreadable_files(diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.pyindex 009ca1e5c..afc86acfb 100644--- a/plugins/codex-security/tests/test_generate_rank_input.py+++ b/plugins/codex-security/tests/test_generate_rank_input.py@@ -847,3 +847,32 @@ def test_make_rank_input_decodes_bom_marked_utf16_source(tmp_path: Path, mode: srun_repo_cli(command, repo, output, *arguments)assert {row["path"]: row["preview"] for row in read_jsonl(output)} == expected+++def test_revision_preview_uses_the_same_read_window_as_local_files(tmp_path: Path) -> None:+ repo = tmp_path / "repo"+ repo.mkdir()+ initialize_repo(repo)+ source = repo / "example.py"+ source.write_text("def first():\n pass\n")+ git(repo, "add", ".")+ git(repo, "commit", "-qm", "base")+ base = git(repo, "rev-parse", "HEAD")+ source.write_text(+ "def first():\n pass\n#" + "x" * 80_000 + "\ndef outside_window():\n pass\n"+ )+ git(repo, "add", ".")+ git(repo, "commit", "-qm", "long source")+ output = tmp_path / "preview.jsonl"+ run_repo_cli(+ "make-diff-rank-input",+ repo,+ output,+ "--base",+ base,+ "--head",+ "HEAD",+ )+ rows = read_jsonl(output)+ assert "first" in rows[0]["preview"]+ assert "outside_window" not in rows[0]["preview"]diff --git a/plugins/codex-security/tests/test_git_inventory_regressions.py b/plugins/codex-security/tests/test_git_inventory_regressions.pyindex b86ed4089..79ea8914d 100644--- a/plugins/codex-security/tests/test_git_inventory_regressions.py+++ b/plugins/codex-security/tests/test_git_inventory_regressions.py@@ -175,6 +175,36 @@ def test_dirty_submodule_warning_preserves_the_changed_target_detail(tmp_path: Passert "results were saved" in warning+def test_committed_binary_detection_agrees_beyond_preview_window(tmp_path: Path) -> None:+ target = tmp_path / "target"+ initialize_git_repository(target)+ base = git(target, "rev-parse", "HEAD").decode()+ (target / "payload.php").write_bytes(b"<?php\n" + b" " * (70 * 1024) + b"\0payload")+ (target / "visible.py").write_text("value = 1\n")+ git(target, "add", ".")+ git(target, "commit", "-qm", "Add source and binary fixtures")+ inventory_path = tmp_path / "inventory.txt"+ rank_path = tmp_path / "rank.jsonl"+ load_script("generate_in_scope_files").generate_diff_in_scope_files(+ target, base, "HEAD", "revisions", inventory_path+ )+ load_script("generate_rank_input").make_diff_rank_input(+ argparse.Namespace(+ repo=str(target),+ base=base,+ head="HEAD",+ mode="revisions",+ out=str(rank_path),+ area="fixture",+ preview_bytes=1024,+ )+ )+ assert inventory_path.read_text().splitlines() == ["visible.py"]+ assert [json.loads(line)["path"] for line in rank_path.read_text().splitlines()] == [+ "visible.py"+ ]++def test_disabled_git_source_snapshot_does_not_inspect_refs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.tsindex 45d6a9753..f32a3bf11 100644--- a/sdk/typescript/tests-ts/multiscan.test.ts+++ b/sdk/typescript/tests-ts/multiscan.test.ts@@ -3474,11 +3474,21 @@ describe("multiscan", () => {});});-test.each(["sha1", "sha256"])(- "bulk checkout preserves %s object format despite Git defaults",- async (objectFormat) => {- const name = `bulk checkout preserves ${objectFormat} object format despite Git defaults`;- if (runTestInSubprocess(fileURLToPath(import.meta.url), name)) return;+test.each([+ ["sha1", "sha1"],+ ["sha1", "sha256"],+ ["sha256", "sha1"],+ ["sha256", "sha256"],+])(+ "bulk checkout preserves %s object format with a %s default",+ async (objectFormat, defaultFormat) => {+ if (+ runTestInSubprocess(+ fileURLToPath(import.meta.url),+ `bulk checkout preserves ${objectFormat} object format with a ${defaultFormat} default`,+ )+ )+ return;const paths = await fixture();const repo = await repository(paths.root, "source", objectFormat);await writeFile(@@ -3486,24 +3496,75 @@ test.each(["sha1", "sha256"])(`id,repository,revision\nfixture,${repo.path},${repo.revision}\n`,);const run = mock(async (checkout: string, scanOptions = {}) => {+ expect(process.env["GIT_DEFAULT_HASH"]).toBe(defaultFormat);expect(git(checkout, "rev-parse", "--show-object-format")).toBe(objectFormat,);+ expect(git(checkout, "rev-parse", "HEAD")).toBe(repo.revision);return completeRun(checkout, scanOptions);});const previous = process.env["GIT_DEFAULT_HASH"];- process.env["GIT_DEFAULT_HASH"] =- objectFormat === "sha1" ? "sha256" : "sha1";+ process.env["GIT_DEFAULT_HASH"] = defaultFormat;try {const result = await runMultiscan(options(paths, client(run)));expect(result.failed).toBe(0);expect(run).toHaveBeenCalledTimes(1);- expect(process.env["GIT_DEFAULT_HASH"]).toBe(- objectFormat === "sha1" ? "sha256" : "sha1",- );+ expect(process.env["GIT_DEFAULT_HASH"]).toBe(defaultFormat);} finally {if (previous === undefined) delete process.env["GIT_DEFAULT_HASH"];else process.env["GIT_DEFAULT_HASH"] = previous;}},);++testPosix(+ "bulk SHA-1 checkout works without Git object-format options",+ async () => {+ if (+ runTestInSubprocess(+ fileURLToPath(import.meta.url),+ "bulk SHA-1 checkout works without Git object-format options",+ )+ )+ return;+ const paths = await fixture();+ const source = await repository(paths.root, "source");+ await writeFile(+ paths.input,+ `id,repository,revision\nfixture,${source.path},${source.revision}\n`,+ );+ const command = await resolveTrustedExecutable(+ "git",+ process.env,+ process.cwd(),+ );+ expect(command).not.toBeNull();+ const bin = await temporaryDirectory();+ await writeFile(+ join(bin, "git"),+ `#!/bin/sh+for argument do+ case "$argument" in+ --object-format=*) echo 'error: unknown option object-format' >&2; exit 129 ;;+ esac+done+exec '${command!.executable.replaceAll("'", "'\\''")}' "$@"+`,+ { mode: 0o700 },+ );+ const previousPath = process.env["PATH"];+ process.env["PATH"] = `${bin}:${previousPath ?? ""}`;+ try {+ const run = mock(async (checkout: string, scanOptions = {}) => {+ expect(git(checkout, "rev-parse", "HEAD")).toBe(source.revision);+ return completeRun(checkout, scanOptions);+ });+ const result = await runMultiscan(options(paths, client(run)));+ expect(result.failed).toBe(0);+ expect(run).toHaveBeenCalledTimes(1);+ } finally {+ if (previousPath === undefined) delete process.env["PATH"];+ else process.env["PATH"] = previousPath;+ }+ },+);diff --git a/sdk/typescript/tests-ts/plugin-report-limits.test.ts b/sdk/typescript/tests-ts/plugin-report-limits.test.tsindex 7f1b044d5..4f2617ea9 100644--- a/sdk/typescript/tests-ts/plugin-report-limits.test.ts+++ b/sdk/typescript/tests-ts/plugin-report-limits.test.ts@@ -50,6 +50,7 @@ describe("bundled scan report and source limits", () => {"diagnostics = {'rootCause': {'summary': 'Missing authorization check.'}, 'validation': {'summary': 'An untrusted request reaches the protected resource.'}, 'attackPath': {'narrative': 'The request bypasses the authorization boundary.'}, 'codeEvidence': [{'id': 'evidence', 'label': 'Missing check', 'path': 'example.py', 'startLine': 1, 'code': 'return resource', 'explanation': 'No authorization check runs.'}], 'evidence': 'The protected resource was exposed.', 'evidenceExcerpt': 'return resource'}","details = {'remediationTests': [f'test-{index}' for index in range(40)], 'preventiveControls': [f'control-{index}' for index in range(40)]}","large = {**diagnostics, 'preventiveControls': ['x' * 900 for _ in range(20)], 'remediationTests': ['Verify authorization.'], 'writeup': {'reportPath': 'findings/example/example.md'}, 'provenance': {'source': 'scan'}, 'severity': {'level': 'high', 'rationale': 'Verified impact'}, 'status': 'open', 'taxonomy': {'category': 'injection', 'cwe': ['CWE-79']}}",+ "oversized_severity = {**large, 'severity': {'level': 'high', 'rationale': 'x' * 18850}}","code_evidence = [{'id': f'evidence-{index}', 'label': 'example', 'path': 'example.py', 'startLine': 1, 'code': 'c' * 1500, 'explanation': 'e' * 1500} for index in range(4)]","rich = {'rootCause': {'summary': 'r' * 2000}, 'validation': {'summary': 'v' * 3000}, 'attackPath': {'narrative': 'a' * 4000}, 'codeEvidence': code_evidence, 'evidenceExcerpt': 'e' * 8000, 'identity': {'anchor': 'finding'}, 'preventiveControls': ['Centralize authorization.'], 'remediationTests': ['Verify authorization.']}","boundary = {**diagnostics, 'remediationTests': ['x'] * 4000, 'preventiveControls': ['Keep authorization centralized.']}",@@ -59,7 +60,7 @@ describe("bundled scan report and source limits", () => {"oversized_metadata = {**diagnostics, 'confidence': {'level': 'high', 'rationale': 'x' * 17000}, 'remediationTests': ['Verify authorization.'], 'preventiveControls': ['Centralize authorization.']}","oversized_guidance = {'rootCause': {'summary': 'root'}, 'validation': {'summary': 'validation'}, 'attackPath': {'narrative': 'attack'}, 'codeEvidence': [{'id': 'evidence', 'label': 'evidence', 'path': 'example.py', 'startLine': 1, 'code': 'x', 'explanation': 'evidence'}], 'evidence': 'legacy', 'evidenceExcerpt': 'excerpt', 'remediationTests': ['x' * 7800], 'preventiveControls': ['y' * 7930]}","nested_boundary = {'remediationTests': ['x'] * 3937, 'rootCause': {'summary': 'r' * 178, 'detail': {'x': {'y': 'z'}}}}",- "projections = {key: bounded_finding_details(value) for key, value in {'details': details, 'large': large, 'rich': rich, 'boundary': boundary, 'unicodeBoundary': unicode_boundary, 'emptyControls': empty_controls, 'emptyTests': empty_tests, 'oversizedMetadata': oversized_metadata, 'oversizedGuidance': oversized_guidance, 'nestedBoundary': nested_boundary}.items()}",+ "projections = {key: bounded_finding_details(value) for key, value in {'details': details, 'large': large, 'rich': rich, 'boundary': boundary, 'unicodeBoundary': unicode_boundary, 'emptyControls': empty_controls, 'emptyTests': empty_tests, 'oversizedMetadata': oversized_metadata, 'oversizedSeverity': oversized_severity, 'oversizedGuidance': oversized_guidance, 'nestedBoundary': nested_boundary}.items()}","print(json.dumps({'projections': projections, 'bytes': {key: len(json.dumps(value, separators=(',', ':')).encode()) for key, value in projections.items()}}))",].join("\n");const result = Bun.spawnSync(@@ -88,6 +89,7 @@ describe("bundled scan report and source limits", () => {remediationTests: string[];preventiveControls: string[];};+ oversizedSeverity: Record<string, unknown>;oversizedMetadata: {remediationTests: string[];preventiveControls: string[];@@ -121,6 +123,9 @@ describe("bundled scan report and source limits", () => {status: "open",taxonomy: { category: "injection", cwe: ["CWE-79"] },});+ expect(projections.oversizedSeverity).toMatchObject({+ writeup: { reportPath: "findings/example/example.md" },+ });expect(projections.rich).toMatchObject({identity: { anchor: "finding" },preventiveControls: ["Centralize authorization."],@@ -134,6 +139,7 @@ describe("bundled scan report and source limits", () => {projections.emptyControls,projections.emptyTests,projections.oversizedMetadata,+ projections.oversizedSeverity,projections.oversizedGuidance,]) {expect(finding).toMatchObject({