Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(targets)!: run inventory and ranking helpers in Node (#1432)

Michael D'Angelo05cceee2479b
diff --git a/github-action/scripts/cli-integration.ts b/github-action/scripts/cli-integration.ts
index 495458e4c..260bd3d78 100644
--- a/github-action/scripts/cli-integration.ts
+++ b/github-action/scripts/cli-integration.ts
@@ -1,6 +1,7 @@
// Exercise the locked CLI or an installed release candidate with synthetic local requests only.
import assert from 'node:assert/strict';
import { execFile, execFileSync, spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
import { copyFile, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises';
import { createServer, type IncomingHttpHeaders } from 'node:http';
import { tmpdir } from 'node:os';
@@ -212,9 +213,14 @@ try {
const scopesFile = join(root, 'literal-scopes.json');
const sourceInput = join(root, 'literal-source.jsonl');
await writeFile(scopesFile, JSON.stringify(scopes));
- execFileSync(python, [join(cliPackage, '_bundled_plugin/scripts/generate_rank_input.py'),
- 'make-repo-scope-input', '--repo', repository, '--scopes-file', scopesFile, '--out', sourceInput],
- {cwd: repository, env, stdio: ['ignore', 'pipe', 'pipe']});
+ const legacyGenerator = join(cliPackage, '_bundled_plugin/scripts/generate_rank_input.py');
+ const usesPythonGenerator = existsSync(legacyGenerator);
+ const generateScopeInput = (output: string, pythonCommand: string, environment: NodeJS.ProcessEnv) =>
+ execFileSync(usesPythonGenerator ? pythonCommand : process.execPath,
+ [usesPythonGenerator ? legacyGenerator : join(cliPackage, '_bundled_plugin/mcp/helpers.mjs'),
+ 'make-repo-scope-input', '--repo', repository, '--scopes-file', scopesFile, '--out', output],
+ {cwd: repository, env: environment, stdio: ['ignore', 'pipe', 'pipe']});
+ generateScopeInput(sourceInput, python, env);
const selectedFiles = (await readFile(sourceInput, 'utf8')).trim().split('\n').map(row => JSON.parse(row).path).sort();
assert.deepEqual(selectedFiles, [...literalFiles].sort(), 'Real scope selection must preserve literal names and exclude glob-matching decoys');
@@ -299,12 +305,10 @@ try {
assert.equal(await readFile(outsideFile, 'utf8'), 'unchanged');
assert.equal(await readFile(workspaceFile, 'utf8'), 'scan output');
}
- // The actual pinned helper must still work when an MCP-style child drops the loader variable.
+ // The bundled helper must still work when an MCP-style child drops the loader variable.
const strippedEnv = {...venvEnv, LD_LIBRARY_PATH:undefined};
const loaderSourceInput = join(root, 'loader-source.jsonl');
- execFileSync(selected.python, [join(cliPackage, '_bundled_plugin/scripts/generate_rank_input.py'),
- 'make-repo-scope-input', '--repo', repository, '--scopes-file', scopesFile, '--out', loaderSourceInput],
- {cwd:repository, env:strippedEnv, stdio:['ignore','pipe','pipe']});
+ generateScopeInput(loaderSourceInput, selected.python, strippedEnv);
assert.equal(await readFile(loaderSourceInput, 'utf8'), await readFile(sourceInput, 'utf8'));
const venvArguments = scanArguments(parseInputs(name => deepInputs[name] ?? '', repository),
{repository}, join(root, 'venv-results'));
diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts
index a05ed29b2..db1b24405 100644
--- a/plugins/codex-security/mcp-app/helpers-main.ts
+++ b/plugins/codex-security/mcp-app/helpers-main.ts
@@ -15,6 +15,7 @@ import {
} from "./src/helpers/normalize-candidates";
import { validatePatchRiskAssessmentCommand } from "./src/helpers/validate-patch-risk-assessment";
import { deepReviewInputCommand } from "./src/helpers/deep-review-input";
+import { inventoryCommand } from "./src/helpers/generate-inventory";
import { bindRepoScopesCommand } from "./src/helpers/bind-repo-scopes";
import { escapeControls, stringifyJson } from "./src/helpers/json";
import { decodeUtf8 } from "./src/helpers/utf8";
@@ -80,7 +81,15 @@ function runHelper(): void {
"read-severity-classification":
"Usage: read-severity-classification --scan-id <id>\nReads a JSON object from stdin with an absolute stateDirectory. Reads saved assessments without updating the database.",
};
- if (command === "resolve-security-md") {
+ if (
+ command === "generate-in-scope-files" ||
+ command === "make-repo-scope-input" ||
+ command === "make-diff-rank-input"
+ ) {
+ void inventoryCommand(command, args).then((code) => {
+ process.exitCode = code;
+ });
+ } else if (command === "resolve-security-md") {
process.exitCode = resolveSecurityMdCommand(args, posixHome);
} else if (command === "normalize-candidates") {
process.exitCode = normalizeCandidatesCommand(args, posixHome);
@@ -136,7 +145,7 @@ function runHelper(): void {
});
} else {
console.error(
- `Usage: launch_codex_security_mcp[.cmd] --helper <resolve-security-md | normalize-candidates | validate-patch-risk-assessment | copy-deep-review-input | bind-repo-scopes | ${Object.keys(workbenchUsage).join(" | ")}> [options]`,
+ `Usage: launch_codex_security_mcp[.cmd] --helper <generate-in-scope-files | make-repo-scope-input | make-diff-rank-input | resolve-security-md | normalize-candidates | validate-patch-risk-assessment | copy-deep-review-input | bind-repo-scopes | ${Object.keys(workbenchUsage).join(" | ")}> [options]`,
);
process.exitCode = 2;
}
diff --git a/plugins/codex-security/mcp-app/src/artifact-inventory.ts b/plugins/codex-security/mcp-app/src/artifact-inventory.ts
index 01d5d3c07..03f777fc1 100644
--- a/plugins/codex-security/mcp-app/src/artifact-inventory.ts
+++ b/plugins/codex-security/mcp-app/src/artifact-inventory.ts
@@ -1,8 +1,6 @@
-import { execFile as nodeExecFile } from "node:child_process";
import { readFile } from "node:fs/promises";
-import { join } from "node:path";
-import { promisify } from "node:util";
import * as z from "zod/v4";
+import { join } from "node:path";
import {
artifactDestination,
artifactSourcePath,
@@ -11,14 +9,11 @@ import {
type ArtifactPage,
} from "./artifact-io.js";
import { artifactSchemaDocuments } from "./artifact-contracts.js";
-import {
- missingPythonHelperMessage,
- resolvePythonCommand,
-} from "./python_command.js";
import { decodeUtf8 } from "./helpers/utf8.js";
-import { decodePosixBytes } from "./helpers/posix-path.js";
+import { runTool } from "./helpers/inventory-git";
+import { environmentValue } from "./helpers/helper-files";
+import { decodePosixBytes, encodePosixPath } from "./helpers/posix-path";
-const execFile = promisify(nodeExecFile);
const inventoryComponents = ["artifacts", "02_discovery", "in_scope_files.txt"];
const label = "review_items";
@@ -48,20 +43,12 @@ export async function prepareCodexSecurityReviewItems(
inventoryComponents,
label,
);
- const pythonCommand = context.pythonCommand ?? (await resolvePythonCommand());
- const helper = join(
- context.pluginRoot,
- "scripts",
- "generate_in_scope_files.py",
- );
+ const windows = process.platform === "win32";
const arguments_ = [
- "-B",
- helper,
- "--repo",
- context.repoRoot,
+ "generate-in-scope-files",
+ `--repo=${context.repoRoot}`,
`--scope=${context.scope ?? "."}`,
- "--out",
- destination,
+ `--out=${destination}`,
];
if (context.mode === "diff") {
@@ -85,27 +72,43 @@ export async function prepareCodexSecurityReviewItems(
);
}
arguments_.push(
- "--diff-base",
- baseRevision,
- "--diff-head",
- headRevision,
- "--diff-mode",
- kind === "working_tree" ? "local-patch" : "revisions",
+ `--diff-base=${baseRevision}`,
+ `--diff-head=${headRevision}`,
+ `--diff-mode=${kind === "working_tree" ? "local-patch" : "revisions"}`,
);
}
try {
- await execFile(pythonCommand, arguments_, {
- cwd: context.pluginRoot,
- encoding: "utf8",
- shell: false,
- });
+ if (arguments_.some((argument) => argument.includes("\0")))
+ throw new TypeError("Process arguments must not contain NUL bytes");
+ const helper = join(context.pluginRoot, "mcp", "helpers.mjs");
+ const home = environmentValue("HOME");
+ const input = windows
+ ? undefined
+ : Buffer.from(
+ encodePosixPath(
+ [home === undefined ? "" : "1", home ?? "", ...arguments_, ""].join(
+ "\0",
+ ),
+ ).toString("hex"),
+ );
+ const result = await runTool(
+ process.execPath,
+ windows ? [helper, ...arguments_] : [helper, "--helper"],
+ // Relative NODE_OPTIONS preloads use the MCP process's current directory.
+ undefined,
+ input,
+ windows ? 0 : 3,
+ );
+ if (result.status !== 0)
+ throw new Error(
+ result.stderr.trim() ||
+ (result.signal
+ ? `Inventory helper terminated by ${result.signal}`
+ : `Inventory helper exited with status ${result.status}`),
+ );
} catch (error) {
- const missingPython = missingPythonHelperMessage(error, pythonCommand);
- if (missingPython) {
- throw new Error(`${label}: ${missingPython}`, { cause: error });
- }
- const details = helperError(error);
+ const details = error instanceof Error ? error.message : String(error);
throw new Error(
`${label}: the scan inventory helper failed${details ? `: ${details}` : "."}`,
{ cause: error },
@@ -163,13 +166,3 @@ async function readReviewItems(
return rows;
}
-
-function helperError(error: unknown): string {
- if (error && typeof error === "object" && "stderr" in error) {
- const stderr = error.stderr;
- if (typeof stderr === "string" && stderr.trim()) return stderr.trim();
- if (Buffer.isBuffer(stderr) && stderr.length)
- return stderr.toString("utf8").trim();
- }
- return error instanceof Error ? error.message : String(error);
-}
diff --git a/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts
new file mode 100644
index 000000000..45940e7fa
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/helpers/generate-inventory.ts
@@ -0,0 +1,545 @@
+import {
+ environmentValue,
+ isMissingPathError,
+ normalizePath,
+ resolvedPathText as canonical,
+} from "./helper-files";
+import { randomBytes } from "node:crypto";
+import { renameSync, unlinkSync } from "node:fs";
+import { dirname, isAbsolute, parse, relative, sep } from "node:path";
+import { parseArgs } from "node:util";
+import { mkdir, readFile, writeFile } from "./helper-files";
+import { expandHome, windowsFiles, windowsJoin } from "./resolve-security-md";
+import { decodePosixBytes, encodePosixPath } from "./posix-path";
+import { stringifyJson } from "./json";
+import {
+ compare,
+ print,
+ reportCommandError,
+ ArgumentError,
+} from "./rank-worklists";
+import { decodeUtf8 } from "./utf8";
+import {
+ ancestors,
+ append,
+ directory,
+ exists,
+ fsPath,
+ inside,
+ lstat,
+ regular,
+ rejectStreams,
+ stat,
+ sourceMetadata,
+ walk,
+ windows,
+} from "./inventory-paths";
+import {
+ blobSamples,
+ changedPaths,
+ decodeGitPath,
+ directoryPaths,
+ encodeGitPath,
+ git,
+ runRipgrep,
+} from "./inventory-git";
+import { PREVIEW_BYTES, previewForBytes, sampleFile } from "./source-preview";
+
+export type InventoryCommand =
+ "generate-in-scope-files" | "make-repo-scope-input" | "make-diff-rank-input";
+export interface InventoryOptions {
+ repo: string;
+ out: string;
+ scope?: string;
+ scopesFile?: string;
+ area?: string;
+ previewBytes?: number;
+ base?: string;
+ head?: string;
+ mode?: string;
+}
+
+function scopePath(
+ repo: string,
+ scope: string,
+ home: string | undefined,
+ explicit: boolean,
+ rejectLinks: boolean,
+): string {
+ const requested = normalizePath(explicit ? scope : expandHome(scope, home));
+ rejectStreams(requested);
+ const requestedRoot = parse(requested).root;
+ const path = windows
+ ? append(
+ windowsFiles()
+ .absolute(Buffer.from(windowsJoin(repo, requestedRoot), "utf16le"))
+ .toString("utf16le"),
+ requested.slice(requestedRoot.length),
+ )
+ : isAbsolute(requested)
+ ? requested
+ : append(repo, requested);
+ if (rejectLinks) {
+ inside(repo, path);
+ // Do not normalize away a link before checking link/.. scopes.
+ let ancestor = repo;
+ const lexical = path.slice(repo.length + (repo.endsWith(sep) ? 0 : 1));
+ for (const part of lexical.split(windows ? /[\\/]/u : /\//u)) {
+ if (!part || part === ".") continue;
+ if (part === "..") {
+ if (ancestor === repo)
+ throw new Error(`Scope must be inside repo: ${path}`);
+ ancestor = dirname(ancestor);
+ continue;
+ }
+ ancestor = append(ancestor, part);
+ const info = lstat(ancestor);
+ if (
+ info.isSymbolicLink() ||
+ ("isNameSurrogate" in info && info.isNameSurrogate())
+ )
+ throw new Error(
+ `Requested scope must not contain symbolic links: ${ancestor}`,
+ );
+ }
+ }
+ const resolved = canonical(path);
+ inside(repo, resolved);
+ if (!directory(resolved) && !regular(resolved))
+ throw new Error(`Scope path not found: ${resolved}`);
+ return resolved;
+}
+
+async function scopeCandidates(repo: string, scope: string): Promise<string[]> {
+ if (regular(scope)) return [scope];
+ const indexed = await directoryPaths(scope);
+ if (indexed !== undefined) return indexed;
+ let result;
+ try {
+ result = await runRipgrep(
+ [
+ "--files",
+ "--hidden",
+ "--no-require-git",
+ "--null",
+ "--glob",
+ "!**/{.git,.svn}",
+ "--glob",
+ "!**/{.git,.svn}/**",
+ "--",
+ relative(repo, scope) || ".",
+ ],
+ repo,
+ );
+ } catch {
+ const ignoreNames = [".gitignore", ".ignore", ".rgignore"];
+ const isIgnoreFile = (path: string) => {
+ try {
+ return stat(path).isFile();
+ } catch (error) {
+ if (isMissingPathError(error)) return false;
+ throw error;
+ }
+ };
+ let ignored =
+ [...ancestors(repo)].some((path) => exists(append(path, ".git"))) ||
+ [...ancestors(scope)].some((path) => {
+ try {
+ inside(repo, path);
+ } catch {
+ return false;
+ }
+ return ignoreNames.some((name) => isIgnoreFile(append(path, name)));
+ });
+ const files: string[] = [];
+ if (!ignored)
+ for (const path of walk(scope, false, true)) {
+ if (
+ ignoreNames.some((name) => path.endsWith(sep + name)) &&
+ isIgnoreFile(path)
+ ) {
+ ignored = true;
+ break;
+ }
+ files.push(path);
+ }
+ if (ignored)
+ throw new Error(
+ "Could not safely enumerate ignored scoped files without Git or ripgrep.",
+ );
+ return files;
+ }
+ if (result.status !== 0 && result.status !== 1)
+ throw new Error(
+ `Could not enumerate scoped repository files${result.signal ? ` (${result.signal})` : ""}: ${result.stderr.trim()}`,
+ );
+ return decodePosixBytes(result.stdout)
+ .split("\0")
+ .filter(Boolean)
+ .map((path) => append(repo, path));
+}
+
+function loadScopes(path: string): string[] {
+ let scopes: unknown;
+ try {
+ scopes = JSON.parse(decodeUtf8(readFile(path)));
+ } catch {
+ throw new Error(`Unable to read scopes file: ${path}`);
+ }
+ if (
+ !Array.isArray(scopes) ||
+ !scopes.length ||
+ scopes.some((scope: unknown) => typeof scope !== "string" || !scope)
+ )
+ throw new Error(
+ `Scopes file must contain a non-empty JSON string array: ${path}`,
+ );
+ return scopes as string[];
+}
+
+function inventoryOutput(value: string): string {
+ if (!value || value.includes("\0"))
+ throw new Error("--out: expected an inventory file path");
+ if (exists(value) && lstat(value).isSymbolicLink())
+ throw new Error("--out: refusing to replace a symbolic link");
+ const output = canonical(value, false);
+ if (exists(output) && !regular(output))
+ throw new Error(`--out: expected a regular file path: ${output}`);
+ return output;
+}
+
+function writeInventory(output: string, rows: Buffer[]): number {
+ const unique = [
+ ...new Map(rows.map((row) => [row.toString("hex"), row])).values(),
+ ].sort(Buffer.compare);
+ mkdir(dirname(output));
+ const temporary = append(
+ dirname(output),
+ `.${randomBytes(12).toString("hex")}.tmp`,
+ );
+ try {
+ writeFile(temporary, unique, true);
+ if (windows)
+ windowsFiles().rename(
+ Buffer.from(temporary, "utf16le"),
+ Buffer.from(output, "utf16le"),
+ );
+ else renameSync(fsPath(temporary), fsPath(output));
+ } finally {
+ if (exists(temporary)) {
+ if (windows) windowsFiles().unlink(Buffer.from(temporary, "utf16le"));
+ else unlinkSync(fsPath(temporary));
+ }
+ }
+ return unique.length;
+}
+
+export async function generateInventory(
+ command: InventoryCommand,
+ options: InventoryOptions,
+ home = environmentValue("HOME"),
+): Promise<number> {
+ const repo = canonical(expandHome(options.repo, home));
+ if (!directory(repo)) throw new Error(`Repo path not found: ${repo}`);
+ const outputPath = expandHome(options.out, home),
+ inventory = command === "generate-in-scope-files";
+ const output = inventory ? inventoryOutput(outputPath) : outputPath;
+ const rankRows = new Map<
+ string,
+ { path: string; area?: string; preview?: string }
+ >();
+ const inventoryRows: Buffer[] = [];
+ const diff =
+ command === "make-diff-rank-input" ||
+ (inventory && options.base !== undefined);
+ const scope = options.scope ?? ".";
+ const resolveInventoryScope = () => {
+ if (!scope || scope.includes("\0"))
+ throw new Error("--scope: expected a non-empty file or directory");
+ try {
+ return scopePath(repo, scope, home, false, false);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ if (message.startsWith("Scope must be inside repo"))
+ throw new Error(`--scope: path must remain inside --repo: ${scope}`);
+ if (error instanceof Error && "code" in error)
+ throw new Error(`--scope: path does not exist: ${scope}`);
+ throw error;
+ }
+ };
+ if (inventory && !diff) {
+ const absolute = resolveInventoryScope();
+ const requested = isAbsolute(expandHome(scope, home))
+ ? inside(repo, absolute)
+ : scope;
+ const result = await runRipgrep(
+ [
+ "--files",
+ "--null",
+ "--hidden",
+ "--path-separator",
+ "/",
+ "--glob",
+ "!**/{.git,.svn}",
+ "--glob",
+ "!**/{.git,.svn}/**",
+ "--",
+ requested,
+ ],
+ repo,
+ ).catch((error: unknown) => {
+ throw new Error(
+ `could not run ripgrep: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ });
+ if (result.status !== 0 && result.status !== 1)
+ throw new Error(
+ `ripgrep ${result.signal ? `terminated by ${result.signal}` : `exited with status ${result.status}`}${result.stderr.trim() ? `: ${result.stderr.trim()}` : ""}`,
+ );
+ let bytes = result.stdout;
+ if ([...ancestors(repo)].some((path) => exists(append(path, ".git")))) {
+ const tracked = await git(repo, [
+ "ls-files",
+ "--cached",
+ "--ignored",
+ "--exclude-standard",
+ "-z",
+ "--",
+ requested,
+ ]).catch((error: unknown) => {
+ if (error instanceof Error && "code" in error)
+ return { status: 127, stdout: Buffer.alloc(0) };
+ throw error;
+ });
+ if (tracked.status === 0) {
+ const prefix =
+ requested === "." || requested.startsWith("./") ? "./" : "";
+ const paths = decodeGitPath(tracked.stdout)
+ .split("\0")
+ .filter((path) => path && regular(append(repo, path)));
+ bytes = Buffer.concat([
+ bytes,
+ encodeGitPath(paths.map((path) => prefix + path + "\0").join("")),
+ ]);
+ }
+ }
+ for (const path of decodePosixBytes(bytes).split("\0").filter(Boolean)) {
+ if (path.split("/").some((part) => sourceMetadata.has(part))) continue;
+ if (/[\r\n]/u.test(path))
+ throw new Error(
+ "Repository contains a path that cannot fit in the file inventory",
+ );
+ inventoryRows.push(encodePosixPath(path + "\n"));
+ }
+ } else if (diff) {
+ if (
+ inventory &&
+ ![".", "./"].includes(
+ isAbsolute(expandHome(scope, home))
+ ? inside(repo, scopePath(repo, scope, home, false, false))
+ : scope,
+ )
+ )
+ throw new Error("--scope: diff scans must use the repository root");
+ const mode = options.mode ?? "revisions",
+ base = options.base!,
+ head = options.head ?? "HEAD";
+ const changes = await changedPaths(repo, base, head, mode).catch(
+ (error: unknown) => {
+ if (inventory)
+ throw new Error(
+ `could not resolve the selected Git changes: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ throw error;
+ },
+ );
+ const refs = changes.filter(
+ ({ status }) => mode === "revisions" || status === "D",
+ );
+ const samples = new Map(refs.map((change, index) => [change.path, index]));
+ const blobs = await blobSamples(
+ repo,
+ refs.map(({ path, status }) => `${status === "D" ? base : head}:${path}`),
+ );
+ for (const { path, status } of changes) {
+ let preview = "";
+ if (samples.has(path)) {
+ const sample = blobs[samples.get(path)!];
+ if (!sample)
+ throw new Error(
+ `Unable to read committed diff blob: ${status === "D" ? base : head}:${path}`,
+ );
+ if (sample[1]) continue;
+ if (!inventory && status !== "D")
+ preview = previewForBytes(sample[0], options.previewBytes)[0];
+ } else {
+ const absolute = append(repo, path);
+ if (!regular(absolute)) continue;
+ let contained = true;
+ try {
+ inside(repo, canonical(absolute));
+ } catch {
+ if (inventory) continue;
+ contained = false;
+ }
+ if (contained) {
+ const [sample, binary] = sampleFile(absolute);
+ if (binary) continue;
+ if (!inventory)
+ preview = previewForBytes(sample, options.previewBytes)[0];
+ }
+ }
+ if (inventory) {
+ const encoded = encodeGitPath(path);
+ if (/[\r\n]/u.test(path))
+ throw new Error(
+ "Git changes contain a path that cannot fit in the file inventory",
+ );
+ inventoryRows.push(Buffer.concat([encoded, Buffer.from("\n")]));
+ } else
+ rankRows.set(path, { path, area: options.area ?? "diff", preview });
+ }
+ } else {
+ const scopes = loadScopes(expandHome(options.scopesFile!, home));
+ const resolved = scopes.map((value) =>
+ scopePath(repo, value, home, true, true),
+ );
+ for (const selected of resolved) {
+ for (const path of await scopeCandidates(repo, selected)) {
+ let name: string;
+ try {
+ if (!regular(path)) continue;
+ name = inside(repo, canonical(path));
+ } catch {
+ continue;
+ }
+ if (name.split("/").some((part) => sourceMetadata.has(part))) continue;
+ rankRows.set(name, { path: name });
+ }
+ }
+ }
+ if (inventory) return writeInventory(output, inventoryRows);
+ mkdir(dirname(output));
+ writeFile(
+ output,
+ [...rankRows.values()]
+ .sort((left, right) => compare(left.path, right.path))
+ .map((row) =>
+ Buffer.from(
+ stringifyJson(row, 0).replace(
+ /[\u007f-\uffff]/g,
+ (unit) => `\\u${unit.charCodeAt(0).toString(16).padStart(4, "0")}`,
+ ) + (windows ? "\r\n" : "\n"),
+ ),
+ ),
+ );
+ return rankRows.size;
+}
+
+export async function inventoryCommand(
+ command: InventoryCommand,
+ args: string[],
+ home = environmentValue("HOME"),
+): Promise<number> {
+ const inventory = command === "generate-in-scope-files",
+ diff = command === "make-diff-rank-input",
+ scoped = command === "make-repo-scope-input";
+ const required = [
+ "repo",
+ "out",
+ ...(inventory
+ ? ["scope"]
+ : diff
+ ? ["base"]
+ : scoped
+ ? ["scopes-file"]
+ : []),
+ ];
+ const optional = inventory
+ ? ["diff-base", "diff-head", "diff-mode"]
+ : diff
+ ? ["head", "mode", "area", "preview-bytes"]
+ : [];
+ const usage = `usage: launch_codex_security_mcp[.cmd] --helper ${command} ${required.map((name) => `--${name} VALUE`).join(" ")}`;
+ try {
+ let values: Record<string, string | boolean | undefined>;
+ try {
+ const normalizedArgs = args.flatMap((value, index) =>
+ value === "--preview-bytes" && /^-\d+$/u.test(args[index + 1] ?? "")
+ ? [`${value}=${args[index + 1]}`]
+ : index > 0 &&
+ args[index - 1] === "--preview-bytes" &&
+ /^-\d+$/u.test(value)
+ ? []
+ : [value],
+ );
+ values = parseArgs({
+ args: normalizedArgs,
+ options: {
+ ...Object.fromEntries(
+ [...required, ...optional].map((name) => [
+ name,
+ { type: "string" as const },
+ ]),
+ ),
+ help: { type: "boolean", short: "h" },
+ },
+ }).values;
+ if (values.help) {
+ print(
+ `${usage}\n\nGenerate deterministic repository file inventories and bounded source previews.\n\n${[...required, ...optional].map((name) => ` --${name} VALUE`).join("\n")}
+
+Directory scopes honor tool ignore rules; explicitly requested files are retained.
+Defaults: scope=., head=HEAD, mode=revisions, preview-bytes=1024; area uses the scope or diff.`,
+ );
+ return 0;
+ }
+ if (required.some((name) => values[name] === undefined))
+ throw new Error(
+ `Missing required options: ${required
+ .filter((name) => values[name] === undefined)
+ .map((name) => `--${name}`)
+ .join(", ")}`,
+ );
+ const mode = values["diff-mode"] ?? values.mode;
+ if (mode !== undefined && mode !== "revisions" && mode !== "local-patch")
+ throw new Error("mode must be revisions or local-patch");
+ if (
+ values["preview-bytes"] !== undefined &&
+ !/^[+-]?\d+$/u.test(String(values["preview-bytes"]).trim())
+ )
+ throw new Error("--preview-bytes must be an integer");
+ } catch (error) {
+ throw new ArgumentError(
+ error instanceof Error ? error.message : String(error),
+ );
+ }
+ const count = await generateInventory(
+ command,
+ {
+ repo: values.repo as string,
+ out: values.out as string,
+ scope: values.scope as string | undefined,
+ scopesFile: values["scopes-file"] as string | undefined,
+ area: values.area as string | undefined,
+ previewBytes:
+ values["preview-bytes"] === undefined
+ ? PREVIEW_BYTES
+ : Number(values["preview-bytes"]),
+ base: (values["diff-base"] ?? values.base) as string | undefined,
+ head: (values["diff-head"] ?? values.head) as string | undefined,
+ mode: (values["diff-mode"] ?? values.mode) as string | undefined,
+ },
+ home,
+ );
+ print(
+ inventory
+ ? `Recorded ${count} in-scope files.`
+ : `Wrote ${count} ${scoped ? "scoped paths" : "rows"} to ${expandHome(values.out as string, home)}`,
+ );
+ return 0;
+ } catch (error) {
+ const code = reportCommandError(error, command, usage);
+ return inventory ? 2 : code;
+ }
+}
diff --git a/plugins/codex-security/mcp-app/src/helpers/helper-files.ts b/plugins/codex-security/mcp-app/src/helpers/helper-files.ts
index 611197f45..dc9ce06d2 100644
--- a/plugins/codex-security/mcp-app/src/helpers/helper-files.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/helper-files.ts
@@ -103,7 +103,7 @@ export function readFile(path: string | number, label?: string): Buffer {
}
}
-function* readChunks(path: string, label?: string): Iterable<Buffer> {
+export function* readChunks(path: string, label?: string): Iterable<Buffer> {
if (process.platform === "win32") {
yield readFile(path, label);
return;
@@ -154,12 +154,24 @@ export function mkdir(path: string): void {
else mkdirSync(encodePosixPath(path), { recursive: true });
}
-export function writeFile(path: string, chunks: Iterable<Buffer>): void {
+export function writeFile(
+ path: string,
+ chunks: Iterable<Buffer>,
+ exclusive = false,
+): void {
if (process.platform === "win32") {
- windowsFileSystem(windowsBinding()).writeFile(widePath(path), chunks);
+ windowsFileSystem(windowsBinding()).writeFile(
+ widePath(path),
+ chunks,
+ exclusive,
+ );
return;
}
- const descriptor = openSync(encodePosixPath(path), "w");
+ const descriptor = openSync(
+ encodePosixPath(path),
+ exclusive ? "wx" : "w",
+ exclusive ? 0o600 : undefined,
+ );
try {
for (const chunk of chunks) writeFileSync(descriptor, chunk);
} finally {
diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
new file mode 100644
index 000000000..5c9e81ddd
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/helpers/inventory-git.ts
@@ -0,0 +1,686 @@
+import {
+ environmentValue,
+ resolvedPathText as canonical,
+} from "./helper-files";
+import {
+ spawn,
+ type SpawnOptions,
+ type ChildProcessWithoutNullStreams,
+} from "node:child_process";
+import { randomUUID } from "node:crypto";
+import type { Writable } from "node:stream";
+import {
+ basename,
+ dirname,
+ delimiter,
+ isAbsolute,
+ resolve,
+ sep,
+} from "node:path";
+import { createRequire } from "node:module";
+import { constants } from "node:os";
+import { spawnWindowsProcess } from "../../../native/windows-process.mjs";
+import { nativeTarget } from "../../../native/platform.mjs";
+import { decodePosixBytes, encodePosixPath } from "./posix-path";
+import { decodeUtf8 } from "./utf8";
+import {
+ ancestors,
+ append,
+ executable,
+ exists,
+ inside,
+ linkedParent,
+ lstat,
+ regular,
+ sameFile,
+ sourceMetadata,
+ walk,
+ windows,
+} from "./inventory-paths";
+import { createSourceSampler } from "./source-preview";
+
+interface ToolResult {
+ status: number | null;
+ signal: NodeJS.Signals | null;
+ stdout: Buffer;
+ stderr: string;
+}
+
+const repositoryEnvironment = [
+ "GIT_ALTERNATE_OBJECT_DIRECTORIES",
+ "GIT_CEILING_DIRECTORIES",
+ "GIT_COMMON_DIR",
+ "GIT_DIR",
+ "GIT_DISCOVERY_ACROSS_FILESYSTEM",
+ "GIT_INDEX_FILE",
+ "GIT_NAMESPACE",
+ "GIT_OBJECT_DIRECTORY",
+ "GIT_WORK_TREE",
+];
+const posixLoaderSetting = /^(?:LD|DYLD)_[A-Z0-9_]+$/u;
+
+function rawToolValue(value: string): boolean {
+ return (windows ? /[\ud800-\udfff]/u : /[\udc80-\udcff]/u).test(value);
+}
+
+function selectedBundledRipgrep(candidate: string): boolean {
+ const codex = environmentValue("CODEX_CLI_PATH");
+ if (!codex) return false;
+ try {
+ return sameFile(
+ candidate,
+ append(
+ append(dirname(dirname(canonical(codex))), "codex-path"),
+ "rg.exe",
+ ),
+ );
+ } catch {
+ return false;
+ }
+}
+
+function trustedTool(
+ name: "git" | "rg",
+ target: string,
+ rawWindowsLookup = false,
+): string | undefined {
+ let protectedRoot = canonical(target);
+ for (const ancestor of ancestors(protectedRoot))
+ if (exists(append(ancestor, ".git"))) protectedRoot = ancestor;
+ const configured =
+ name === "git" ? environmentValue("CODEX_SECURITY_GIT") : undefined;
+ if (configured === "") return undefined;
+ if (configured !== undefined && !isAbsolute(configured))
+ throw new Error(
+ "CODEX_SECURITY_GIT must name an absolute trusted executable.",
+ );
+ const searchPath =
+ environmentValue("PATH") ?? (windows ? ".;C:\\bin" : "/usr/bin:/bin");
+ const directories =
+ windows && name === "rg"
+ ? windowsToolPaths(searchPath)
+ : searchPath.split(delimiter);
+ const candidates =
+ configured !== undefined
+ ? [configured]
+ : directories.flatMap((entry) => {
+ const directory =
+ windows && name === "git" ? entry.replace(/^"|"$/gu, "") : entry;
+ let nativeWindowsLookup = rawWindowsLookup;
+ if (windows && name === "rg") {
+ try {
+ nativeWindowsLookup ||= rawToolValue(
+ canonical(resolve(target, directory)),
+ );
+ } catch {
+ return [];
+ }
+ }
+ const names = windows
+ ? name === "rg" && !nativeWindowsLookup
+ ? ["rg.com", "rg.exe"]
+ : [`${name}.exe`, `${name}.com`]
+ : [name];
+ return names.map((name) => append(directory, name));
+ });
+ for (const selected of candidates) {
+ const candidate =
+ name === "rg" &&
+ (!isAbsolute(selected) || (windows && /^[\\/](?![\\/])/u.test(selected)))
+ ? windows
+ ? resolve(target, selected)
+ : append(target, selected)
+ : selected;
+ let invocation: string, resolved: string;
+ try {
+ invocation = append(canonical(dirname(candidate)), basename(candidate));
+ resolved = canonical(candidate);
+ } catch {
+ continue;
+ }
+ if (
+ !executable(resolved) ||
+ (windows &&
+ (!/\.(exe|com)$/iu.test(candidate) || /\.(bat|cmd)$/iu.test(resolved)))
+ )
+ continue;
+ const inRepository = [resolve(candidate), invocation, resolved].some(
+ (path) => {
+ try {
+ inside(protectedRoot, path);
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ );
+ if (inRepository && !(name === "rg" && selectedBundledRipgrep(resolved))) {
+ if (configured !== undefined)
+ throw new Error(
+ "CODEX_SECURITY_GIT must stay outside the protected repository.",
+ );
+ continue;
+ }
+ return invocation;
+ }
+ return undefined;
+}
+
+function windowsToolPaths(searchPath: string): string[] {
+ const paths: string[] = [];
+ for (let start = 0; start < searchPath.length;) {
+ let end = start;
+ const quote = searchPath[start];
+ if (quote === '"' || quote === "'") {
+ end = searchPath.indexOf(quote, start + 1);
+ if (end < 0) end = searchPath.length;
+ }
+ end = searchPath.indexOf(";", end);
+ if (end < 0) end = searchPath.length;
+ const entry = searchPath.slice(start, end).replace(/^["']|["']$/gu, "");
+ if (entry) paths.push(entry);
+ start = end + 1;
+ }
+ return paths;
+}
+
+function spawnTool(
+ command: string,
+ args: string[],
+ options: SpawnOptions & {
+ cwd?: string;
+ stdio: ["ignore" | "pipe", "pipe", "pipe"];
+ },
+ inheritedEnvironment: Record<string, string>,
+ inputFd: 0 | 3 = 0,
+) {
+ const stdio = [...options.stdio];
+ if (inputFd === 3) stdio.push("pipe");
+ const environment = Object.fromEntries(
+ Object.entries(inheritedEnvironment).filter(
+ ([name, value]) =>
+ rawToolValue(value) || (!windows && posixLoaderSetting.test(name)),
+ ),
+ );
+ const values = [
+ command,
+ ...args,
+ ...(typeof options.cwd === "string" ? [options.cwd] : []),
+ ...Object.values(environment),
+ ];
+ if (!values.some(rawToolValue)) {
+ try {
+ return {
+ child: spawn(command, args, { ...options, stdio }),
+ failureMarker: undefined,
+ };
+ } catch (error) {
+ const failure = error as NodeJS.ErrnoException;
+ // macOS Node exposes ENOEXEC numerically; use the shell's text-file fallback.
+ if (
+ windows ||
+ (failure.code !== "ENOEXEC" &&
+ failure.errno !== -constants.errno.ENOEXEC)
+ )
+ throw error;
+ }
+ }
+ if (windows) {
+ const binary = createRequire(import.meta.url).resolve(
+ `./native/${nativeTarget}/windows.node`,
+ );
+ return {
+ child: spawnWindowsProcess(binary, command, args, options, environment),
+ failureMarker: undefined,
+ };
+ }
+ // Node encodes argv/cwd as UTF-8. POSIX sh can reconstruct raw filename bytes
+ // with its builtin printf; a sentinel preserves trailing newlines in each value.
+ const assign = (value: string) => {
+ if (value.includes("\0"))
+ throw new TypeError("Process arguments must not contain NUL bytes");
+ const octal = [...encodePosixPath(value)]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ return `value=$(printf '%b.' '${octal}'); value=\${value%.}`;
+ };
+ const failureMarker = `\0${randomUUID()}\0`;
+ const reportFailure = `printf '${failureMarker.replaceAll("\0", "\\000")}' >&2`;
+ const script = [
+ "set --",
+ ...Object.entries(environment).flatMap(([name, value]) => [
+ assign(value),
+ `${name}="$value"; export ${name}`,
+ ]),
+ ...[command, ...args].flatMap((value) => [
+ assign(value),
+ 'set -- "$@" "$value"',
+ ]),
+ ...(typeof options.cwd === "string"
+ ? [assign(options.cwd), 'cd -P -- "$value" || exit']
+ : []),
+ // Only a failed exec emits this launch-specific frame. A tool may itself
+ // exit 126/127, and a separate status pipe can leak through shell redirects.
+ 'if [ -n "${BASH_VERSION:-}" ]; then shopt -s execfail; fi',
+ `trap "${reportFailure}" 0`,
+ 'exec "$@"',
+ // Bash 3.2 clears EXIT traps on failed exec; dash exits through the trap.
+ "status=$?",
+ "trap - 0",
+ reportFailure,
+ 'exit "$status"',
+ ].join("\n");
+ // Loader settings belong to the tool: decoded raw values can fail at shell
+ // startup, and macOS system shells also strip ordinary DYLD_* values.
+ const env = { ...(options.env ?? process.env) };
+ for (const name of Object.keys(environment)) delete env[name];
+ // macOS ships dash; unlike its Bash 3.2 sh, failed exec retains ENOENT's 127.
+ const shell = process.platform === "darwin" ? "/bin/dash" : "/bin/sh";
+ const child = spawn(shell, ["-c", script], {
+ ...options,
+ env,
+ cwd: undefined,
+ stdio,
+ });
+ return { child, failureMarker };
+}
+
+function inheritedToolPaths(additional: string[] = []): Record<string, string> {
+ const keys = Object.keys(process.env).sort();
+ const toolConfiguration = keys.filter((name) => {
+ const key = windows ? name.toUpperCase() : name;
+ return (
+ [
+ "GIT_CONFIG_GLOBAL",
+ "GIT_CONFIG_SYSTEM",
+ "GIT_CONFIG_PARAMETERS",
+ ].includes(key) ||
+ /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/u.test(key) ||
+ // Restore loader settings before exec: the tool cannot repair them after
+ // its dynamic loader has already tried to open libraries and preload files.
+ (!windows && posixLoaderSetting.test(key))
+ );
+ });
+ return Object.fromEntries(
+ [
+ "HOME",
+ "PATH",
+ "XDG_CONFIG_HOME",
+ ...additional,
+ ...toolConfiguration,
+ ].flatMap((name) => {
+ const key = windows
+ ? (keys.find((key) => key.toUpperCase() === name.toUpperCase()) ?? name)
+ : name;
+ const value = environmentValue(key);
+ return value === undefined ? [] : [[name, value]];
+ }),
+ );
+}
+
+function gitProcess(repo: string, args: string[]) {
+ const command = trustedTool("git", repo);
+ if (!command) return undefined;
+ const env: NodeJS.ProcessEnv = { ...process.env };
+ const inherited = inheritedToolPaths();
+ for (const name of Object.keys(env)) {
+ const key = windows ? name.toUpperCase() : name;
+ if (repositoryEnvironment.includes(key) || key === "GIT_LITERAL_PATHSPECS")
+ delete env[name];
+ }
+ env.GIT_LITERAL_PATHSPECS = "1";
+ return spawnTool(
+ command,
+ [
+ "-c",
+ "core.fsmonitor=false",
+ "-c",
+ "i18n.logOutputEncoding=UTF-8",
+ "-C",
+ repo,
+ ...args,
+ ],
+ { env, stdio: ["pipe", "pipe", "pipe"] },
+ inherited,
+ );
+}
+
+export async function runRipgrep(
+ args: string[],
+ repo: string,
+): Promise<ToolResult> {
+ if (!windows) return runTool("rg", args, repo);
+ const inherited = inheritedToolPaths([
+ "RIPGREP_CONFIG_PATH",
+ "CODEX_SECURITY_GIT",
+ ]);
+ const rawWindowsLookup = [repo, ...args, ...Object.values(inherited)].some(
+ rawToolValue,
+ );
+ const command = trustedTool("rg", repo, rawWindowsLookup);
+ if (!command)
+ throw Object.assign(new Error("spawn rg ENOENT"), { code: "ENOENT" });
+ return runTool(command, args, repo);
+}
+
+export async function runTool(
+ command: string,
+ args: string[],
+ cwd?: string,
+ input?: Buffer,
+ inputFd: 0 | 3 = 0,
+): Promise<ToolResult> {
+ const execution = spawnTool(
+ command,
+ args,
+ {
+ cwd,
+ stdio: [
+ input === undefined || inputFd !== 0 ? "ignore" : "pipe",
+ "pipe",
+ "pipe",
+ ],
+ },
+ inheritedToolPaths(["RIPGREP_CONFIG_PATH", "CODEX_SECURITY_GIT"]),
+ inputFd,
+ );
+ const pipe = execution.child.stdio[inputFd] as Writable | null;
+ pipe?.on("error", () => {});
+ pipe?.end(input);
+ return collect(execution);
+}
+
+async function collect({
+ child,
+ failureMarker,
+}: ReturnType<typeof spawnTool>): Promise<ToolResult> {
+ const stdout: Buffer[] = [],
+ stderr: Buffer[] = [];
+ child.stdout!.on("data", (chunk: Buffer) => stdout.push(chunk));
+ child.stderr!.on("data", (chunk: Buffer) => stderr.push(chunk));
+ await new Promise<void>((resolve, reject) => {
+ child.once("error", reject);
+ child.once("close", () => resolve());
+ });
+ const diagnostic = Buffer.concat(stderr);
+ const start = failureMarker ? diagnostic.indexOf(failureMarker) : -1;
+ if (start >= 0)
+ throw Object.assign(
+ new Error(
+ Buffer.concat([
+ diagnostic.subarray(0, start),
+ diagnostic.subarray(start + failureMarker!.length),
+ ]).toString("utf8"),
+ ),
+ { code: child.exitCode === 127 ? "ENOENT" : "EACCES" },
+ );
+ return {
+ status: child.exitCode,
+ signal: child.signalCode,
+ stdout: Buffer.concat(stdout),
+ stderr: diagnostic.toString("utf8"),
+ };
+}
+
+export async function git(repo: string, args: string[]) {
+ try {
+ const execution = gitProcess(repo, args);
+ if (execution) {
+ execution.child.stdin!.end();
+ return await collect(execution);
+ }
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+ }
+ return { status: 127, signal: null, stdout: Buffer.alloc(0), stderr: "" };
+}
+
+export function decodeGitPath(data: Buffer): string {
+ if (!windows) return decodePosixBytes(data);
+ // Git index paths can contain UTF-8-encoded lone UTF-16 code units on Windows.
+ return data
+ .toString("latin1")
+ .split(/(\xed[\xa0-\xbf][\x80-\xbf])/u)
+ .map((part) =>
+ /^\xed[\xa0-\xbf][\x80-\xbf]$/u.test(part)
+ ? String.fromCharCode(
+ 0xd000 |
+ ((part.charCodeAt(1) & 0x3f) << 6) |
+ (part.charCodeAt(2) & 0x3f),
+ )
+ : decodeUtf8(Buffer.from(part, "latin1")),
+ )
+ .join("");
+}
+
+export function encodeGitPath(value: string): Buffer {
+ if (!windows) return encodePosixPath(value);
+ return Buffer.concat(
+ value.split(/([\ud800-\udfff])/u).map((part) => {
+ if (!/^[\ud800-\udfff]$/u.test(part)) return Buffer.from(part);
+ const unit = part.charCodeAt(0);
+ return Buffer.from([
+ 0xe0 | (unit >> 12),
+ 0x80 | ((unit >> 6) & 0x3f),
+ 0x80 | (unit & 0x3f),
+ ]);
+ }),
+ );
+}
+
+function gitLine(data: Buffer): string {
+ return decodeGitPath(data).replace(windows ? /\r?\n$/u : /\n$/u, "");
+}
+
+function paths(data: Buffer): string[] {
+ return decodeGitPath(data).split("\0").filter(Boolean);
+}
+function requireSuccess(result: ToolResult): Buffer {
+ if (result.status !== 0)
+ throw new Error(
+ result.stderr.trim() ||
+ (result.signal
+ ? `Git terminated by ${result.signal}`
+ : `Git exited with status ${result.status}`),
+ );
+ return result.stdout;
+}
+
+export async function directoryPaths(
+ target: string,
+): Promise<string[] | undefined> {
+ const root = await git(target, ["rev-parse", "--show-toplevel"]);
+ if (root.status !== 0 || !root.stdout.length) return undefined;
+ const repository = canonical(gitLine(root.stdout));
+ const prefix = decodeGitPath(
+ requireSuccess(await git(target, ["rev-parse", "--show-prefix"])),
+ )
+ .replace(/\n$/u, "")
+ .replace(/\/$/u, "");
+ const scope = prefix ? append(repository, prefix) : repository;
+ inside(repository, canonical(scope));
+ if (!sameFile(scope, target))
+ throw new Error("Scan target must stay inside its Git working tree.");
+ const depth = prefix ? prefix.split("/").length : 0;
+ // Git's ASCII case folding does not cover Unicode directory aliases.
+ const unicodeCase = [...prefix].some(
+ (character) =>
+ character.charCodeAt(0) > 127 &&
+ character.toLowerCase() !== character.toUpperCase(),
+ );
+ const listing = paths(
+ requireSuccess(
+ await git(repository, [
+ ...(depth && !unicodeCase ? ["--no-literal-pathspecs"] : []),
+ "ls-files",
+ "--cached",
+ "--others",
+ "--exclude-standard",
+ "-z",
+ "--",
+ depth && !unicodeCase ? `:(icase,literal)${prefix}` : ".",
+ ]),
+ ),
+ );
+ const found = new Set<string>(),
+ matching = new Map<string, boolean>();
+ for (const name of listing) {
+ const parts = name.split("/");
+ if (parts.some((part) => sourceMetadata.has(part))) continue;
+ if (parts.length <= depth) continue;
+ if (depth) {
+ const indexedPrefix = append(repository, parts.slice(0, depth).join(sep));
+ if (!matching.has(indexedPrefix))
+ matching.set(indexedPrefix, sameFile(indexedPrefix, scope));
+ if (!matching.get(indexedPrefix)) continue;
+ }
+ const path = append(target, parts.slice(depth).join(sep));
+ let metadata;
+ try {
+ if (linkedParent(target, path)) continue;
+ metadata = lstat(path);
+ } catch (error) {
+ const code = (error as NodeJS.ErrnoException).code;
+ if (code === "ENOENT" || code === "ENOTDIR") continue;
+ throw error;
+ }
+ found.add(path);
+ if (
+ metadata.isDirectory() &&
+ !("isNameSurrogate" in metadata && metadata.isNameSurrogate())
+ ) {
+ const nestedRoot = await git(path, ["rev-parse", "--show-toplevel"]);
+ const nested =
+ nestedRoot.status === 0 && sameFile(gitLine(nestedRoot.stdout), path)
+ ? await directoryPaths(path)
+ : undefined;
+ for (const child of nested ?? walk(path, true, true)) found.add(child);
+ }
+ }
+ return [...found];
+}
+
+export interface Change {
+ path: string;
+ status: string;
+}
+async function changed(repo: string, args: string[]): Promise<Change[]> {
+ const fields = paths(
+ requireSuccess(
+ await git(repo, [
+ "diff",
+ "--ignore-submodules=all",
+ "--raw",
+ "-z",
+ "--diff-filter=ACMRDT",
+ ...args,
+ "--",
+ ]),
+ ),
+ );
+ const changes: Change[] = [];
+ for (let index = 0; index < fields.length;) {
+ const metadata = fields[index++]!.split(" "),
+ status = metadata.at(-1)![0]!;
+ if (status === "C" || status === "R") index++;
+ const path = fields[index++]!;
+ if (
+ (status === "D" ? metadata[0]!.slice(1) : metadata[1]!).startsWith("100")
+ )
+ changes.push({ path, status });
+ }
+ return changes;
+}
+export async function changedPaths(
+ repo: string,
+ base: string,
+ head: string,
+ mode: string,
+): Promise<Change[]> {
+ if (mode === "revisions") return changed(repo, [`${base}..${head}`]);
+ const unstaged = await changed(repo, [base]),
+ staged = await changed(repo, ["--cached", base]);
+ const result = new Map(staged.map((change) => [change.path, change]));
+ for (const change of unstaged) result.set(change.path, change);
+ for (const path of paths(
+ requireSuccess(
+ await git(repo, ["ls-files", "--others", "--exclude-standard", "-z"]),
+ ),
+ ))
+ if (!path.endsWith("/")) result.set(path, { path, status: "A" });
+ return [...result.values()].filter(
+ ({ path, status }) =>
+ !path.split("/").some((part) => sourceMetadata.has(part)) &&
+ (status === "D" || regular(append(repo, path))),
+ );
+}
+
+/** Consume each complete blob while retaining at most the preview prefix. */
+export async function blobSamples(
+ repo: string,
+ names: string[],
+): Promise<([Buffer, boolean] | undefined)[]> {
+ if (!names.length) return [];
+ const nulInput = names.some((name) => /[\r\n]/u.test(name));
+ const separator = nulInput ? "\0" : "\n";
+ const child = gitProcess(repo, [
+ "cat-file",
+ "--batch",
+ ...(nulInput ? ["-z"] : []),
+ ])?.child as ChildProcessWithoutNullStreams | undefined;
+ if (!child) return names.map(() => undefined);
+ const completion = new Promise<number>((resolve, reject) => {
+ child.once("error", reject);
+ child.once("close", (code) => resolve(code ?? 1));
+ });
+ child.stderr.resume();
+ child.stdin.on("error", () => {});
+ child.stdin.end(encodeGitPath(names.join(separator) + separator));
+ const result: ([Buffer, boolean] | undefined)[] = [];
+ let pending = Buffer.alloc(0),
+ remaining: number | undefined,
+ sampler = createSourceSampler();
+ try {
+ for await (const chunk of child.stdout) {
+ pending = Buffer.concat([pending, chunk as Buffer]);
+ while (pending.length) {
+ if (remaining === undefined) {
+ const end = pending.indexOf(0x0a);
+ if (end < 0) break;
+ const header = pending.subarray(0, end).toString("utf8").split(" ");
+ pending = pending.subarray(end + 1);
+ if (header.at(-2) !== "blob") {
+ result.push(undefined);
+ continue;
+ }
+ remaining = Number(header.at(-1));
+ sampler = createSourceSampler();
+ }
+ if (remaining > 0) {
+ const length = Math.min(remaining, pending.length),
+ data = pending.subarray(0, length);
+ sampler.consume(data);
+ remaining -= length;
+ pending = pending.subarray(length);
+ if (remaining) break;
+ }
+ if (!pending.length) break;
+ if (pending[0] !== 0x0a) throw new Error("Invalid Git blob framing");
+ pending = pending.subarray(1);
+ result.push(sampler.finish());
+ remaining = undefined;
+ }
+ }
+ if (
+ (await completion) ||
+ remaining !== undefined ||
+ result.length !== names.length
+ )
+ return names.map(() => undefined);
+ return result;
+ } catch (error) {
+ child.kill();
+ await completion.catch(() => {});
+ throw error;
+ }
+}
diff --git a/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts b/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts
new file mode 100644
index 000000000..cecc2c27a
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/helpers/inventory-paths.ts
@@ -0,0 +1,148 @@
+import {
+ accessSync,
+ constants,
+ lstatSync,
+ readdirSync,
+ statSync,
+} from "node:fs";
+import {
+ dirname,
+ isAbsolute,
+ parse,
+ relative,
+ sep,
+ toNamespacedPath,
+} from "node:path";
+import { decodePosixBytes, encodePosixPath } from "./posix-path";
+import { windowsFiles } from "./resolve-security-md";
+import { isMissingPathError } from "./helper-files";
+
+export const windows = process.platform === "win32";
+export const fsPath = (path: string) =>
+ windows ? toNamespacedPath(path) : encodePosixPath(path);
+export const append = (parent: string, child: string) =>
+ `${parent || "."}${parent.endsWith(sep) ? "" : sep}${child}`;
+export const stat = (path: string) =>
+ windows
+ ? windowsFiles().stat(Buffer.from(path, "utf16le"))
+ : statSync(fsPath(path));
+export const lstat = (path: string) =>
+ windows
+ ? windowsFiles().stat(Buffer.from(path, "utf16le"), false)
+ : lstatSync(fsPath(path));
+export function exists(path: string): boolean {
+ try {
+ lstat(path);
+ return true;
+ } catch {
+ return false;
+ }
+}
+export function regular(path: string): boolean {
+ try {
+ return !lstat(path).isSymbolicLink() && stat(path).isFile();
+ } catch (error) {
+ if (isMissingPathError(error)) return false;
+ throw error;
+ }
+}
+export function directory(path: string): boolean {
+ try {
+ return stat(path).isDirectory();
+ } catch {
+ return false;
+ }
+}
+export function inside(root: string, path: string): string {
+ const result = relative(root, path);
+ if (isAbsolute(result) || result === ".." || result.startsWith(`..${sep}`))
+ throw new Error(`Scope must be inside repo: ${path}`);
+ return result.split(sep).join("/") || ".";
+}
+export function rejectStreams(path: string): void {
+ if (!windows) return;
+ const root = parse(path).root;
+ const stream = path
+ .slice(root.length)
+ .split(/[\\/]/u)
+ .find((part) => part.includes(":"));
+ if (stream)
+ throw new Error(
+ `Scope must not use an NTFS alternate data stream: ${stream}`,
+ );
+}
+export function* ancestors(path: string): Iterable<string> {
+ for (let current = path; ; current = dirname(current)) {
+ yield current;
+ if (dirname(current) === current) return;
+ }
+}
+export function sameFile(left: string, right: string): boolean {
+ try {
+ if (windows) {
+ const a = windowsFiles().identity(Buffer.from(left, "utf16le")),
+ b = windowsFiles().identity(Buffer.from(right, "utf16le"));
+ return a.volume === b.volume && a.fileId.equals(b.fileId);
+ }
+ const a = statSync(fsPath(left), { bigint: true }),
+ b = statSync(fsPath(right), { bigint: true });
+ return a.dev === b.dev && a.ino === b.ino;
+ } catch {
+ return false;
+ }
+}
+export function linkedParent(scope: string, path: string): boolean {
+ const parts = relative(scope, path).split(sep);
+ let parent = scope;
+ for (const part of parts.slice(0, -1)) {
+ parent = append(parent, part);
+ const info = lstat(parent);
+ if (
+ info.isSymbolicLink() ||
+ ("isNameSurrogate" in info && info.isNameSurrogate())
+ )
+ return true;
+ }
+ return false;
+}
+export const sourceMetadata = new Set([".git", ".svn"]);
+export function* walk(
+ path: string,
+ skipMetadata = false,
+ ignoreDirectoryReadErrors = false,
+): Iterable<string> {
+ let names: Buffer[];
+ try {
+ names = windows
+ ? windowsFiles()
+ .entriesWithTypes(Buffer.from(path, "utf16le"))
+ .map((entry) => entry.name)
+ : readdirSync(fsPath(path), { encoding: "buffer" });
+ } catch (error) {
+ if (!ignoreDirectoryReadErrors) throw error;
+ return;
+ }
+ const children = names
+ .map((name) =>
+ windows ? name.toString("utf16le") : decodePosixBytes(name),
+ )
+ .filter((name) => !skipMetadata || !sourceMetadata.has(name))
+ .map((name) => append(path, name));
+ yield* children;
+ for (const child of children) {
+ const info = lstat(child);
+ if (
+ info.isDirectory() &&
+ !("isNameSurrogate" in info && info.isNameSurrogate())
+ )
+ yield* walk(child, skipMetadata, ignoreDirectoryReadErrors);
+ }
+}
+export function executable(path: string): boolean {
+ try {
+ if (!windows) accessSync(fsPath(path), constants.X_OK);
+ return stat(path).isFile();
+ } catch {
+ return false;
+ }
+}
diff --git a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts
index 8e3c1a690..a14f0ee2c 100644
--- a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts
+++ b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts
@@ -17,7 +17,7 @@ export interface RankRow {
preview?: string;
}
const blank = (value: string) => value.trim().length === 0;
-function compare(left: string, right: string): number {
+export function compare(left: string, right: string): number {
const a = Array.from(left, (character) => character.codePointAt(0)!);
const b = Array.from(right, (character) => character.codePointAt(0)!);
for (let index = 0; index < Math.min(a.length, b.length); index++) {
diff --git a/plugins/codex-security/mcp-app/src/helpers/source-preview.ts b/plugins/codex-security/mcp-app/src/helpers/source-preview.ts
new file mode 100644
index 000000000..a1c203d95
--- /dev/null
+++ b/plugins/codex-security/mcp-app/src/helpers/source-preview.ts
@@ -0,0 +1,141 @@
+import { windowsFiles } from "./resolve-security-md";
+import { decodePosixBytes } from "./posix-path";
+import { readChunks } from "./helper-files";
+
+export const PREVIEW_BYTES = 1024;
+export const PREVIEW_READ_BYTES = 64 * 1024;
+
+const bomEncoding = (data: Buffer) =>
+ data[0] === 0xff && data[1] === 0xfe
+ ? "utf-16le"
+ : data[0] === 0xfe && data[1] === 0xff
+ ? "utf-16be"
+ : "utf-8";
+
+function decodeSource(data: Buffer): string {
+ // Preview decoding historically ignores malformed or incomplete source units.
+ if (bomEncoding(data) === "utf-8")
+ return decodePosixBytes(data)
+ .replace(/^[\ufeff]/u, "")
+ .replace(/[\udc80-\udcff]/gu, "");
+ const units = Buffer.from(data.subarray(2, data.length - (data.length % 2)));
+ if (bomEncoding(data) === "utf-16be") units.swap16();
+ return units.toString("utf16le").replace(/[\ud800-\udfff]/gu, "");
+}
+
+export function isBinarySample(data: Buffer): boolean {
+ return bomEncoding(data) === "utf-8"
+ ? data.includes(0)
+ : decodeSource(data).includes("\0");
+}
+
+export function truncateUtf8(text: string, budget: number): string {
+ if (budget <= 0) return "";
+ const bytes = Buffer.from(text);
+ if (bytes.length <= budget) return text;
+ let end = Math.min(budget, bytes.length);
+ while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end--;
+ return bytes.subarray(0, end).toString("utf8");
+}
+
+export function previewForBytes(
+ data: Buffer,
+ budget = PREVIEW_BYTES,
+): [string, boolean] {
+ if (isBinarySample(data)) return ["", true];
+ if (budget <= 0) return ["", false];
+ let lines = decodeSource(data).replace(/\r\n?/gu, "\n").split("\n");
+ let first = 0,
+ last = lines.length;
+ while (first < last && !lines[first]!.trim()) first++;
+ while (last > first && !lines[last - 1]!.trim()) last--;
+ lines = lines.slice(first, last);
+ const complete = lines.join("\n");
+ if (Buffer.byteLength(complete) <= budget) return [complete, false];
+ const nonblank = lines.filter((line) => line.trim());
+ const remainder = nonblank.slice(12);
+ lines =
+ remainder.length <= 10
+ ? nonblank
+ : [
+ ...nonblank.slice(0, 12),
+ "...",
+ ...Array.from(
+ { length: 10 },
+ (_, index) =>
+ remainder[Math.floor((index * (remainder.length - 1)) / 9)]!,
+ ),
+ ];
+ if (!lines.length) return ["", false];
+ const sampled = lines.join("\n");
+ if (Buffer.byteLength(sampled) <= budget) return [sampled, false];
+ const render = (size: number) =>
+ lines
+ .map((line) => (line === "..." ? line : truncateUtf8(line, size)))
+ .join("\n");
+ let low = 0,
+ high = Math.max(...lines.map((line) => Buffer.byteLength(line))),
+ best = "";
+ while (low <= high) {
+ const middle = Math.floor((low + high) / 2),
+ candidate = render(middle);
+ if (Buffer.byteLength(candidate) <= budget) {
+ best = candidate;
+ low = middle + 1;
+ } else high = middle - 1;
+ }
+ return [
+ best.trim() && best.trim() !== "..." ? best : truncateUtf8(sampled, budget),
+ false,
+ ];
+}
+
+/** Share bounded preview and binary classification across file and Git reads. */
+export function createSourceSampler() {
+ let sample = Buffer.alloc(0),
+ binary = false,
+ bom = Buffer.alloc(0),
+ unitTail = Buffer.alloc(0);
+ return {
+ consume(data: Buffer): boolean {
+ if (sample.length < PREVIEW_READ_BYTES)
+ sample = Buffer.concat([
+ sample,
+ data.subarray(0, PREVIEW_READ_BYTES - sample.length),
+ ]);
+ const classified = Buffer.concat([unitTail, data]);
+ if (!bom.length && sample.length >= 2 && bomEncoding(sample) !== "utf-8")
+ bom = sample.subarray(0, 2);
+ // Hold one byte until encoding can be identified, and preserve UTF-16 unit alignment.
+ const classifyLength =
+ classified.length -
+ (bom.length || sample.length < 2 ? classified.length % 2 : 0);
+ if (classifyLength)
+ binary ||= bom.length
+ ? isBinarySample(
+ Buffer.concat([bom, classified.subarray(0, classifyLength)]),
+ )
+ : classified.subarray(0, classifyLength).includes(0);
+ unitTail = classified.subarray(classifyLength);
+ return !binary;
+ },
+ finish(): [Buffer, boolean] {
+ if (!bom.length && unitTail.length) binary ||= unitTail.includes(0);
+ return [binary ? Buffer.alloc(0) : sample, binary];
+ },
+ };
+}
+
+/** Retain a bounded preview while checking every byte for binary content. */
+export function sampleFile(path: string): [Buffer, boolean] {
+ try {
+ const sampler = createSourceSampler();
+ if (process.platform === "win32")
+ windowsFiles().readChunks(Buffer.from(path, "utf16le"), sampler.consume);
+ else
+ for (const chunk of readChunks(path)) if (!sampler.consume(chunk)) break;
+ return sampler.finish();
+ } catch {
+ return [Buffer.alloc(0), true];
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts b/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts
index 1d1bacc8e..f439485e9 100644
--- a/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts
+++ b/plugins/codex-security/mcp-app/tests/support/fake-codex-source.ts
@@ -1,3 +1,4 @@
+import path from "node:path";
import { jsonLineInputSource } from "./json-line-input-source.ts";
export function fakeCodexSource(
@@ -5,7 +6,7 @@ export function fakeCodexSource(
preflightAllowed: boolean,
accountResult: unknown,
preflightMarkerPath: string,
-) {
+): string {
return `#!/usr/bin/env node
import { existsSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
@@ -16,7 +17,7 @@ const accountResult = ${JSON.stringify(accountResult)};
const preflightMarkerPath = process.env.FAKE_CODEX_PREFLIGHT_MARKER ?? ${JSON.stringify(preflightMarkerPath)};
const temporaryEnvironment = Object.fromEntries(['TEMP', 'TMP'].map(name => [name, process.env[name]]));
if (process.argv.includes('app-server')) {
- const preflight = { argv: process.argv.slice(2), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, stateDirectory: process.env.CODEX_SECURITY_STATE_DIR, temporaryEnvironment, runnerTrackingId: process.env.RUNNER_TRACKING_ID, libraryPath: process.env.LD_LIBRARY_PATH, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), requests: [] };
+ const preflight = { argv: process.argv.slice(2), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, stateDirectory: process.env.CODEX_SECURITY_STATE_DIR, temporaryEnvironment, runnerTrackingId: process.env.RUNNER_TRACKING_ID, libraryPath: process.env.LD_LIBRARY_PATH, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'HOME', 'NODE_OPTIONS', 'NODE_DEBUG', 'NODE_EXTRA_CA_CERTS', 'CODEX_MCP_NODE_PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), requests: [] };
writeFileSync(preflightMarkerPath, JSON.stringify(preflight));
${jsonLineInputSource(` if (message.method === 'initialized') continue;
if (message.method === 'config/read' || message.method === 'permissionProfile/list') {
@@ -64,7 +65,7 @@ const profileIndex = process.argv.indexOf('--profile');
const profileContents = profileIndex === -1 ? undefined : readFileSync(join(process.env.CODEX_HOME, process.argv[profileIndex + 1] + '.config.toml'), 'utf8');
const openaiAuthentication = stdin.includes('CAPTURE_SYNTHETIC_OPENAI_AUTH') ? { OPENAI_API_KEY: process.env.OPENAI_API_KEY, CODEX_API_KEY: process.env.CODEX_API_KEY } : undefined;
const bedrockAuthentication = stdin.includes('CAPTURE_SYNTHETIC_BEDROCK_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_BEDROCK_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined;
-const runtimeEnvironment = Object.fromEntries(['PATH', 'HOME', 'PYTHON', 'PYTHONUTF8', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'DYLD_FALLBACK_LIBRARY_PATH', 'CODEX_SECURITY_STATE_DIR', 'RUNNER_TRACKING_ID'].map(name => [name, process.env[name]]));
+const runtimeEnvironment = Object.fromEntries(['PATH', 'HOME', 'PYTHON', 'PYTHONUTF8', 'CODEX_MCP_NODE_PATH', 'CODEX_BROWSER_USE_NODE_PATH', 'CODEX_ELECTRON_RESOURCES_PATH', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'DYLD_FALLBACK_LIBRARY_PATH', 'CODEX_SECURITY_STATE_DIR', 'RUNNER_TRACKING_ID'].map(name => [name, process.env[name]]));
const toolProbe = stdin.includes('CAPTURE_SYNTHETIC_RG') ? spawnSync('rg', ['--version'], { encoding: 'utf8' }) : undefined;
if (toolProbe && toolProbe.status !== 0) throw toolProbe.error ?? new Error(toolProbe.stderr);
const pythonProbe = stdin.includes('CAPTURE_SYNTHETIC_PYTHON') ? spawnSync(process.env.PYTHON, ['-I', '-c', 'import json,os,sys; print(json.dumps([sys.prefix,os.environ.get("LD_LIBRARY_PATH")]))'], { encoding: 'utf8' }) : undefined;
@@ -73,7 +74,7 @@ const pythonRuntime = pythonProbe ? JSON.parse(pythonProbe.stdout) : undefined;
const knowledgePath = stdin.includes('synthetic worker configuration fixture') ? process.env.CODEX_SECURITY_KNOWLEDGE_BASE : undefined;
const knowledgeDocuments = knowledgePath === undefined ? undefined : Object.fromEntries(readdirSync(knowledgePath).map(name => [name, readFileSync(join(knowledgePath, name), 'utf8')]));
const deepConfig = process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH ? readFileSync(process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, 'utf8') : undefined;
-writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ ...(scopeInput === undefined ? {} : { scopeInput }), argv: process.argv.slice(2), stdin, cwd: process.cwd(), knowledgePath, knowledgeDocuments, deepConfig, codexHome: process.env.CODEX_HOME, temporaryEnvironment, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, mcpNodePath: process.env.CODEX_MCP_NODE_PATH, bundledTool: toolProbe?.stdout.trim(), pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));
+writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ ...(scopeInput === undefined ? {} : { scopeInput }), argv: process.argv.slice(2), stdin, cwd: process.cwd(), knowledgePath, knowledgeDocuments, deepConfig, codexHome: process.env.CODEX_HOME, temporaryEnvironment, cacheDirectory: process.env.XDG_CACHE_HOME, gitEnvironment: Object.fromEntries(['PATH', 'HOME', 'NODE_OPTIONS', 'NODE_DEBUG', 'NODE_EXTRA_CA_CERTS', 'CODEX_MCP_NODE_PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, mcpNodePath: process.env.CODEX_MCP_NODE_PATH, bundledTool: toolProbe?.stdout.trim(), pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, providerHeader: process.env.SYNTHETIC_HEADER_VALUE, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(profileContents === undefined ? {} : { profileContents }), ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));
if (stdin.includes('COMPLETE_THEN_HANG')) process.on('SIGTERM', () => { if (!stdin.includes('IGNORE_TERMINATION')) setTimeout(() => process.exit(0), 100); });
if (stdin.includes('THREAD_START_CONFIG_ERROR')) { console.error('Error: thread/start: thread/start failed: agents.max_threads cannot be set when features.multi_agent_v2 is enabled (code -32600)'); process.exit(1); }
@@ -130,3 +131,60 @@ if (stdin.includes('COMPLETE_THEN_HANG')) { setInterval(() => {}, 1_000); await
}
`;
}
+
+export function workerGitEnvironment(root: string) {
+ return {
+ NODE_OPTIONS: "--no-warnings",
+ NODE_DEBUG: "synthetic-ambient-debug",
+ NODE_EXTRA_CA_CERTS: path.join(root, "ambient-ca.pem"),
+ CODEX_MCP_NODE_PATH: path.join(root, "inherited node"),
+ HOME: path.join(root, "inherited home"),
+ PATH: [path.join(root, "selected tools"), "../selected tools", ""].join(
+ path.delimiter,
+ ),
+ CODEX_SECURITY_GIT: path.join(root, "selected tools", "git"),
+ GIT_SSH_COMMAND: "synthetic-ssh --fixture",
+ GIT_CONFIG_GLOBAL: path.join(root, "operator.gitconfig"),
+ };
+}
+
+export function workerScanEnvironment(root: string, index: number) {
+ return {
+ HOME: path.join(root, `scan home ${index}`),
+ PATH: path.join(root, `scan tools ${index}`),
+ NODE_OPTIONS: `--stack-trace-limit=${20 + index}`,
+ NODE_DEBUG: `synthetic-scan-debug-${index}`,
+ NODE_EXTRA_CA_CERTS: path.join(root, `scan-ca-${index}.pem`),
+ CODEX_MCP_NODE_PATH: path.join(root, `scan node ${index}`),
+ };
+}
+
+export const expectedGitEnvironment = (
+ gitEnvironment: ReturnType<typeof workerGitEnvironment>,
+ nodePath: string,
+ environment?: ReturnType<typeof workerScanEnvironment>,
+) => ({
+ ...gitEnvironment,
+ CODEX_MCP_NODE_PATH: nodePath,
+ ...(environment === undefined
+ ? {}
+ : {
+ HOME: environment.HOME,
+ PATH: environment.PATH,
+ NODE_OPTIONS: environment.NODE_OPTIONS,
+ NODE_DEBUG: environment.NODE_DEBUG,
+ NODE_EXTRA_CA_CERTS: environment.NODE_EXTRA_CA_CERTS,
+ CODEX_MCP_NODE_PATH: environment.CODEX_MCP_NODE_PATH,
+ }),
+});
+
+export function restoreEnv(name: string, value: string | undefined) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+}
+
+export function nativeConfigOverrides(args: readonly string[]) {
+ return args.flatMap((arg, index) =>
+ arg === "--config" || arg === "-c" ? [args[index + 1]] : [],
+ );
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts b/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts
index 8a4e6321b..5a5cdbf9c 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_discovery.ts
@@ -488,6 +488,11 @@ async function verifyDiffInventoryAllowsDeletedFiles() {
return git("rev-parse", "HEAD");
};
git("init", "-q");
+ const metadata = [".svn/entries", "nested/.svn/entries"];
+ for (const name of [...metadata, ".svn-backup/support.ts"]) {
+ await mkdir(path.dirname(path.join(repository, name)), { recursive: true });
+ await writeFile(path.join(repository, name), "synthetic saved source\n");
+ }
await writeFile(
path.join(repository, "deleted.ts"),
"one\rtwo\r\nthree\nfour",
@@ -554,6 +559,9 @@ async function verifyDiffInventoryAllowsDeletedFiles() {
git("reset", "--hard", baseRevision);
await rm(path.join(repository, "deleted.ts"));
await writeFile(path.join(repository, "changed.ts"), "current\n");
+ git("rm", "--", metadata[0]!);
+ await rm(path.join(repository, metadata[1]!));
+ await rm(path.join(repository, ".svn-backup/support.ts"));
}
const scan: ArtifactContext = {
...context,
@@ -592,6 +600,23 @@ async function verifyDiffInventoryAllowsDeletedFiles() {
path.dirname(inventory),
"candidate_ledger.jsonl",
);
+ if (kind === "working_tree") {
+ await record(scan, [
+ ["changed.ts", 1],
+ [".svn-backup/support.ts", 1],
+ ]);
+ for (const name of metadata) {
+ const accepted = await readFile(destination, "utf8");
+ await assert.rejects(
+ record(scan, [
+ ["changed.ts", 1],
+ [name, 1],
+ ]),
+ );
+ assert.equal(await readFile(destination, "utf8"), accepted);
+ }
+ await record(scan, sources);
+ }
const accepted = await readFile(destination, "utf8");
await assert.rejects(
record(
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts
index c2c31f4fd..c72e4544e 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_inventory.ts
@@ -7,10 +7,12 @@ import {
readdir,
mkdir,
readFile,
+ rename,
symlink,
unlink,
writeFile,
} from "node:fs/promises";
+import { build } from "esbuild";
import path from "node:path";
import { promisify } from "node:util";
import { importSource } from "./import-module.ts";
@@ -20,6 +22,16 @@ const temporaryDirectories = createTemporaryDirectories(true);
const inventory = await importSource(
path.join(import.meta.dirname, "../src/artifact-inventory.ts"),
+ {
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ },
);
try {
@@ -32,6 +44,9 @@ try {
await testPrepareExcludesGitMetadata();
await testPrepareUsesOnlyAuthoritativeDiffChanges();
await testPrepareIncludesStagedAndUnstagedChanges();
+ await testInventoryChildPreservesSettingsAndCwd();
+ if (process.platform !== "win32") await testInventoryChildPreservesRawPaths();
+ await testInventoryChildFailuresPreserveOutput();
await testWorkerReadsItsOwnBoundInventory();
await testCursorDoesNotExceedInventory();
await testEmptyInventoryIsValid();
@@ -40,7 +55,7 @@ try {
await testInventoryReadErrorsAreReported();
await testWorkersCannotPrepareInventory();
await testBoundScopeFailurePreservesPreviousInventory();
- await testPythonLaunchErrors();
+ await testInventoryDoesNotLaunchPython();
await testInvalidDiffTargetPreservesPreviousInventory();
} finally {
await temporaryDirectories.cleanup();
@@ -71,6 +86,7 @@ async function testSchemasAreBoundAndExact() {
async function testPrepareUsesTheExistingStandardGenerator() {
const fixture = await createFixture("standard repository");
await fixture.writeRepositoryFile("src/a.ts", "export const a = 1;\n");
+ await fixture.writeRepositoryFile("-source.py", "source\n");
await fixture.writeRepositoryFile("src/résumé.ts", "export const b = 2;\n");
for (const name of ["\uFEFF来源.ts", "name-\uFFFD.ts", "🙂.ts"])
await fixture.writeRepositoryFile(`src/${name}`, "export {};\n");
@@ -129,6 +145,214 @@ async function testPrepareUsesTheExistingStandardGenerator() {
),
expectedPaths,
);
+ assert.deepEqual(
+ await inventory.prepareCodexSecurityReviewItems({
+ ...fixture.scan,
+ scope: "-source.py",
+ }),
+ { reviewItemsTotal: 1 },
+ );
+ assert.equal(await readFile(fixture.scanInventory, "utf8"), "-source.py\n");
+}
+
+async function testInventoryChildPreservesSettingsAndCwd() {
+ const fixture = await createFixture("inventory child settings");
+ await runGit(fixture.repoRoot, "init", "-q");
+ await runGit(fixture.repoRoot, "commit", "--allow-empty", "-qm", "base");
+ const baseRevision = await runGit(fixture.repoRoot, "rev-parse", "HEAD");
+ await fixture.writeRepositoryFile("visible.py", "visible\n");
+ await fixture.writeRepositoryFile("hidden.py", "hidden\n");
+ const home = path.join(fixture.root, "selected home");
+ const tools = path.join(fixture.root, "selected tools");
+ await mkdir(home);
+ await mkdir(tools);
+ const ignores = path.join(fixture.root, "global-ignore");
+ const config = path.join(fixture.root, "global-config");
+ await writeFile(ignores, "hidden.py\n");
+ await writeFile(
+ config,
+ `[core]\nexcludesFile = ${JSON.stringify(ignores)}\n`,
+ );
+ const marker = path.join(fixture.root, "child.jsonl");
+ const preload = path.join(fixture.root, "preload.cjs");
+ const names = [
+ "HOME",
+ "PATH",
+ "GIT_CONFIG_GLOBAL",
+ "NODE_OPTIONS",
+ "CODEX_MCP_NODE_PATH",
+ "LD_LIBRARY_PATH",
+ "DYLD_LIBRARY_PATH",
+ ];
+ await writeFile(
+ preload,
+ `require('node:fs').appendFileSync(${JSON.stringify(marker)}, JSON.stringify({pid:process.pid,node:process.execPath,cwd:process.cwd(),argv:process.argv,env:Object.fromEntries(${JSON.stringify(names)}.map(name=>[name,process.env[name]]))})+'\\n');`,
+ );
+ const saved = names.map((name) => [name, process.env[name]] as const);
+ const selected = {
+ HOME: home,
+ PATH: tools + path.delimiter + (process.env.PATH ?? ""),
+ GIT_CONFIG_GLOBAL: config,
+ NODE_OPTIONS: `--require ${JSON.stringify(path.relative(process.cwd(), preload).split(path.sep).join("/"))}`,
+ CODEX_MCP_NODE_PATH: path.join(fixture.root, "unused-node"),
+ LD_LIBRARY_PATH: path.join(fixture.root, "selected libraries"),
+ DYLD_LIBRARY_PATH: path.join(fixture.root, "selected dylibs"),
+ };
+ Object.assign(process.env, selected);
+ try {
+ const context = {
+ ...fixture.scan,
+ mode: "diff",
+ targetContract: {
+ diffTarget: {
+ kind: "working_tree",
+ baseRevision,
+ headRevision: "HEAD",
+ },
+ },
+ };
+ assert.deepEqual(await inventory.prepareCodexSecurityReviewItems(context), {
+ reviewItemsTotal: 1,
+ });
+ assert.deepEqual(await inventory.listCodexSecurityReviewItems(context), {
+ items: [{ path: "visible.py" }],
+ });
+ const records = (await readFile(marker, "utf8"))
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ const child = records.find(
+ (record) =>
+ record.argv[1] ===
+ path.join(fixture.scan.pluginRoot, "mcp", "helpers.mjs"),
+ );
+ assert.ok(child);
+ assert.notEqual(child.pid, process.pid);
+ assert.equal(child.node, process.execPath);
+ assert.equal(child.cwd, process.cwd());
+ assert.deepEqual(child.env, selected);
+ for (const [name, value] of Object.entries(selected))
+ assert.equal(process.env[name], value);
+ } finally {
+ for (const [name, value] of saved) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ }
+}
+
+async function testInventoryChildPreservesRawPaths() {
+ const fixture = await createFixture("inventory raw child paths");
+ await runGit(fixture.repoRoot, "init", "-q");
+ await runGit(fixture.repoRoot, "commit", "--allow-empty", "-qm", "base");
+ const baseRevision = await runGit(fixture.repoRoot, "rev-parse", "HEAD");
+ await fixture.writeRepositoryFile("visible.py", "visible\n");
+ await fixture.writeRepositoryFile("hidden.py", "hidden\n");
+ // APFS requires valid UTF-8 names; Linux also permits undecodable bytes.
+ const suffix =
+ process.platform === "darwin" ? Buffer.from("雪") : Buffer.from([0xff]);
+ const suffixText = process.platform === "darwin" ? "雪" : "\udcff";
+ const rawPath = (name: string) =>
+ Buffer.concat([Buffer.from(path.join(fixture.root, name)), suffix]);
+ const repo = rawPath("repository-");
+ await rename(fixture.repoRoot, repo);
+ const ignore = path.join(fixture.root, "global-ignore");
+ await writeFile(ignore, "hidden.py\n");
+ const config = rawPath("global-config-");
+ await writeFile(config, `[core]\nexcludesFile = ${JSON.stringify(ignore)}\n`);
+ const git = rawPath("selected-git-");
+ await writeFile(git, '#!/bin/sh\nexec git "$@"\n', { mode: 0o700 });
+ const context = {
+ ...fixture.scan,
+ repoRoot: path.join(fixture.root, "repository-") + suffixText,
+ mode: "diff",
+ targetContract: {
+ diffTarget: { kind: "working_tree", baseRevision, headRevision: "HEAD" },
+ },
+ };
+ const marker = path.join(fixture.root, "child.json");
+ const preload = path.join(fixture.root, "preload.cjs");
+ const loaderSettings = {
+ LD_LIBRARY_PATH: path.join(fixture.root, "selected libraries"),
+ DYLD_LIBRARY_PATH: path.join(fixture.root, "selected dylibs"),
+ };
+ await writeFile(
+ preload,
+ `const fs = require('node:fs'); fs.writeFileSync(${JSON.stringify(marker)}, JSON.stringify({stdin:fs.readFileSync(0).length,env:Object.fromEntries(${JSON.stringify(Object.keys(loaderSettings))}.map(name=>[name,process.env[name]]))}));`,
+ );
+ const driver = path.join(fixture.root, "prepare.mjs");
+ await build({
+ stdin: {
+ contents: `import { prepareCodexSecurityReviewItems } from ${JSON.stringify(path.resolve(import.meta.dirname, "../src/artifact-inventory.ts"))}; Object.assign(process.env, ${JSON.stringify({ ...loaderSettings, NODE_OPTIONS: `--require ${JSON.stringify(preload)}` })}); console.log(JSON.stringify(await prepareCodexSecurityReviewItems(${JSON.stringify(context)})));`,
+ loader: "ts",
+ resolveDir: import.meta.dirname,
+ },
+ bundle: true,
+ platform: "node",
+ format: "esm",
+ outfile: driver,
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ });
+ const octal = (value: Buffer) =>
+ [...value]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ const { stdout } = await execFile(
+ "/bin/sh",
+ [
+ "-c",
+ [
+ // Raw environment bytes are valid on macOS even though APFS names are UTF-8.
+ 'HOME=$(printf "/fixture/home-\\377"); export HOME',
+ `GIT_CONFIG_GLOBAL=$(printf '%b' '${octal(config)}'); export GIT_CONFIG_GLOBAL`,
+ `CODEX_SECURITY_GIT=$(printf '%b' '${octal(git)}'); export CODEX_SECURITY_GIT`,
+ 'exec "$1" "$2"',
+ ].join("\n"),
+ "inventory-fixture",
+ process.execPath,
+ driver,
+ ],
+ { encoding: "utf8" },
+ );
+ assert.deepEqual(JSON.parse(stdout), { reviewItemsTotal: 1 });
+ assert.equal(await readFile(fixture.scanInventory, "utf8"), "visible.py\n");
+ assert.deepEqual(JSON.parse(await readFile(marker, "utf8")), {
+ stdin: 0,
+ env: loaderSettings,
+ });
+}
+
+async function testInventoryChildFailuresPreserveOutput() {
+ for (const signal of process.platform === "win32" ? [false] : [false, true]) {
+ const fixture = await createFixture("inventory child failure");
+ const pluginRoot = path.join(fixture.root, "failing plugin");
+ await mkdir(path.join(pluginRoot, "mcp"), { recursive: true });
+ await writeFile(
+ path.join(pluginRoot, "mcp", "helpers.mjs"),
+ signal
+ ? 'process.kill(process.pid, "SIGTERM");'
+ : 'process.stderr.write("synthetic inventory failure\\n"); process.exitCode = 23;',
+ );
+ await writeInventory(fixture.scanInventory, "previous.py\n");
+ await assert.rejects(
+ inventory.prepareCodexSecurityReviewItems({
+ ...fixture.scan,
+ pluginRoot,
+ }),
+ signal ? /SIGTERM/u : /synthetic inventory failure/u,
+ );
+ assert.equal(
+ await readFile(fixture.scanInventory, "utf8"),
+ "previous.py\n",
+ );
+ }
}
async function testPrepareDoesNotWritePluginBytecode() {
@@ -139,6 +363,13 @@ async function testPrepareDoesNotWritePluginBytecode() {
recursive: true,
filter: (file) => path.basename(file) !== "__pycache__",
});
+ await cp(
+ path.join(fixture.scan.pluginRoot, "mcp"),
+ path.join(pluginRoot, "mcp"),
+ {
+ recursive: true,
+ },
+ );
await fixture.writeRepositoryFile("example.py", "value = 1\n");
const result = await inventory.prepareCodexSecurityReviewItems({
...fixture.scan,
@@ -508,32 +739,42 @@ async function testInvalidDiffTargetPreservesPreviousInventory() {
"src/original.ts\n",
);
}
+ await assert.rejects(
+ inventory.prepareCodexSecurityReviewItems({
+ ...fixture.scan,
+ mode: "diff",
+ targetContract: {
+ diffTarget: {
+ kind: "working_tree",
+ baseRevision: "HEAD\0--out\0unexpected",
+ headRevision: "HEAD",
+ },
+ },
+ }),
+ /NUL/u,
+ );
+ assert.equal(
+ await readFile(fixture.scanInventory, "utf8"),
+ "src/original.ts\n",
+ );
}
-async function testPythonLaunchErrors() {
- const fixture = await createFixture("python launch failure");
+async function testInventoryDoesNotLaunchPython() {
+ const fixture = await createFixture("unused Python selection");
+ await fixture.writeRepositoryFile("source.py", "source\n");
const pythonCommand = path.join(fixture.root, "python");
- for (const code of process.platform === "win32"
- ? ["ENOENT"]
- : ["ENOENT", "EACCES"]) {
- if (code === "EACCES") await writeFile(pythonCommand, "", { mode: 0o600 });
- await assert.rejects(
- inventory.prepareCodexSecurityReviewItems({
+ for (const present of [false, true]) {
+ if (present) await writeFile(pythonCommand, "", { mode: 0o600 });
+ assert.deepEqual(
+ await inventory.prepareCodexSecurityReviewItems({
...fixture.scan,
pythonCommand,
}),
- (error: Error & { cause?: NodeJS.ErrnoException }) => {
- assert.equal(error.cause?.code, code);
- assert.ok(
- error.message.includes(`spawn ${pythonCommand} ${code}`),
- error.message,
- );
- assert.equal(
- error.message.includes("Reinstall or update"),
- code === "ENOENT",
- );
- return true;
- },
+ { reviewItemsTotal: 1 },
+ );
+ assert.equal(
+ await readFile(fixture.scanInventory, "utf8"),
+ "./source.py\n",
);
}
}
@@ -546,7 +787,10 @@ async function createFixture(label: string) {
const repoRoot = path.join(fixtureRoot, "repository");
const scanRoot = path.join(fixtureRoot, "scan");
const workerRoot = path.join(fixtureRoot, "worker");
- const pluginRoot = path.join(import.meta.dirname, "../../");
+ const pluginRoot = path.resolve(
+ import.meta.dirname,
+ "../../../../sdk/typescript/_bundled_plugin",
+ );
await mkdir(repoRoot, { recursive: true });
await mkdir(scanRoot, { recursive: true });
return {
diff --git a/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts b/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts
index a26d8363b..82b0d5253 100644
--- a/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts
+++ b/plugins/codex-security/mcp-app/tests/test_candidate_inventory.ts
@@ -8,7 +8,7 @@ import { importSource } from "./import-module.ts";
import { createTemporaryDirectories } from "./support/temporary-directories.ts";
const { normalizeCandidatesCommand } = await importSource(
- new URL("../src/helpers/normalize-candidates.ts", import.meta.url).pathname,
+ path.join(import.meta.dirname, "../src/helpers/normalize-candidates.ts"),
);
const { recordCodexSecurityDiscoveryCandidates } = await importSource(
path.join(import.meta.dirname, "../src/artifact-discovery.ts"),
@@ -23,9 +23,9 @@ const { recordCodexSecurityDiscoveryCandidates } = await importSource(
},
},
);
-const inventoryScript = path.join(
+const inventoryHelper = path.join(
import.meta.dirname,
- "../../scripts/generate_in_scope_files.py",
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
);
for (const deleted of [false, true]) {
@@ -66,8 +66,9 @@ for (const deleted of [false, true]) {
git("commit", "-qm", "change");
const scope = path.join(root, "in-scope.txt");
- execFileSync(process.env.PYTHON ?? "python3", [
- inventoryScript,
+ execFileSync(process.execPath, [
+ inventoryHelper,
+ "generate-in-scope-files",
"--repo",
repo,
"--scope",
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
index 3fb5bfae2..31f4f6185 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
@@ -1,4 +1,10 @@
import { testArtifactStartupTimeoutClassification } from "./deep_scan_startup_cases.ts";
+import {
+ workerGitEnvironment,
+ workerScanEnvironment,
+ expectedGitEnvironment,
+ restoreEnv,
+} from "./support/fake-codex-source.ts";
import { windowsAcl } from "./support/windows-output-fixture.ts";
import {
createFakeCodexFixture,
@@ -39,6 +45,7 @@ import {
} from "node:fs/promises";
import { createRequire, syncBuiltinESMExports } from "node:module";
import path from "node:path";
+import { rootCertificates } from "node:tls";
import { fileURLToPath } from "node:url";
import { importModule } from "./import-module.ts";
@@ -1453,6 +1460,11 @@ async function testWorkerRuntimeSettings() {
"PYTHON",
"CODEX_SECURITY_PYTHON_COMMAND",
"PATH",
+ "HOME",
+ "NODE_OPTIONS",
+ "NODE_DEBUG",
+ "NODE_EXTRA_CA_CERTS",
+ "CODEX_MCP_NODE_PATH",
"CODEX_SECURITY_GIT",
"GIT_SSH_COMMAND",
"GIT_CONFIG_GLOBAL",
@@ -1502,16 +1514,8 @@ async function testWorkerRuntimeSettings() {
"python",
);
process.env.PYTHON = python;
- const gitEnvironment = {
- PATH: [
- path.join(fixture.root, "selected tools"),
- "../selected tools",
- "",
- ].join(path.delimiter),
- CODEX_SECURITY_GIT: path.join(fixture.root, "selected tools", "git"),
- GIT_SSH_COMMAND: "synthetic-ssh --fixture",
- GIT_CONFIG_GLOBAL: path.join(fixture.root, "operator.gitconfig"),
- };
+ const gitEnvironment = workerGitEnvironment(fixture.root);
+ await writeFile(gitEnvironment.NODE_EXTRA_CA_CERTS, rootCertificates[0]!);
Object.assign(process.env, gitEnvironment);
const configPath = path.join(fixture.root, "active [scan] config.toml");
const codexHome = path.join(
@@ -1716,6 +1720,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
index === 0
? undefined
: {
+ ...workerScanEnvironment(fixture.root, index),
SYNTHETIC_GATEWAY_KEY: providerKeys[index],
SYNTHETIC_HEADER_VALUE: providerHeaders[index],
CODEX_SQLITE_HOME: path.join(
@@ -1844,6 +1849,14 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
),
),
writeFile(entry.path, stringifyToml(entry.configuration)),
+ ...(entry.environment === undefined
+ ? []
+ : [
+ writeFile(
+ entry.environment.NODE_EXTRA_CA_CERTS,
+ rootCertificates[0]!,
+ ),
+ ]),
writeFile(
entry.deepPath,
stringifyToml({
@@ -2042,6 +2055,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
index,
{ model, reasoningEffort },
] of settings.entries()) {
+ const selectedProvider = workerConfigurations[index];
const workerLaunch = workerLaunches.find(({ args }) =>
model === undefined
? !args.includes("--model")
@@ -2162,7 +2176,11 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
path.join(fixture.root, `state-${index}`),
);
assert.equal(invocation.providerKey, providerKeys[index]);
- assert.equal(invocation.mcpNodePath, nodePaths[index]);
+ assert.equal(
+ invocation.mcpNodePath,
+ selectedProvider.environment?.CODEX_MCP_NODE_PATH ??
+ nodePaths[index],
+ );
assert.equal(process.env.CODEX_MCP_NODE_PATH, nodePaths.at(-1));
assert.equal(invocation.providerHeader, providerHeaders[index]);
@@ -2259,7 +2277,6 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
invocation.configPath,
workerConfigurations[index].path,
);
- const selectedProvider = workerConfigurations[index];
if (selectedProvider.provider === undefined) {
assert.equal(
invocation.argv.some(
@@ -2330,8 +2347,18 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
),
false,
);
- assert.deepEqual(invocation.gitEnvironment, gitEnvironment);
- for (const [name, value] of Object.entries(gitEnvironment)) {
+ assert.deepEqual(
+ invocation.gitEnvironment,
+ expectedGitEnvironment(
+ gitEnvironment,
+ nodePaths[index],
+ selectedProvider.environment,
+ ),
+ );
+ for (const [name, value] of Object.entries({
+ ...gitEnvironment,
+ CODEX_MCP_NODE_PATH: nodePaths.at(-1),
+ })) {
assert.equal(process.env[name], value);
}
assert.equal(invocation.python, python);
@@ -2413,7 +2440,6 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
args.includes("app-server"),
)) {
const preflight = await readJson(launch.markerPath);
- assert.deepEqual(preflight.gitEnvironment, gitEnvironment);
const selectedProvider = workerConfigurations.find(
(entry) => preflight.configPath === entry.path,
);
@@ -2421,6 +2447,14 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`,
selectedProvider,
"worker preflight must use the same scan environment",
);
+ assert.deepEqual(
+ preflight.gitEnvironment,
+ expectedGitEnvironment(
+ gitEnvironment,
+ nodePaths[workerConfigurations.indexOf(selectedProvider)],
+ selectedProvider.environment,
+ ),
+ );
const index = workerConfigurations.indexOf(selectedProvider);
assert.equal(preflight.codexHome, await realpath(codexHome));
assert.equal(
@@ -3867,11 +3901,6 @@ function assertWorkerSubagentPolicy(
assert.equal(args.includes("features.multi_agent_v2.enabled=true"), false);
}
-function restoreEnv(name: string, value: string | undefined) {
- if (value === undefined) delete process.env[name];
- else process.env[name] = value;
-}
-
async function withWorkerFixture<Result>(
action: (
fixture: Awaited<ReturnType<typeof fakeCodexFixture>>,
diff --git a/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts b/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts
new file mode 100644
index 000000000..c98e74ecf
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_inventory_paths.ts
@@ -0,0 +1,592 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import fs from "node:fs";
+import childProcess from "node:child_process";
+import { createRequire, syncBuiltinESMExports } from "node:module";
+import { constants, tmpdir } from "node:os";
+import { join } from "node:path";
+import { importSource } from "./import-module.ts";
+import type { WindowsBinding } from "../../native/windows-binding.mjs";
+
+for (const [name, details] of [
+ ["named", { code: "ENOEXEC" }],
+ [
+ "numeric",
+ { code: "Unknown system error -8", errno: -constants.errno.ENOEXEC },
+ ],
+] as const)
+ test(
+ `POSIX tools run executable text after a ${name} exec-format launch error`,
+ { skip: process.platform === "win32" },
+ async () => {
+ const { runTool } = await importSource("src/helpers/inventory-git.ts", {
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ });
+ const root = fs.mkdtempSync(join(tmpdir(), "inventory-exec-format-"));
+ const tool = join(root, "plain tool");
+ const diagnostic =
+ "synthetic tool failure\n\u000000000000-0000-4000-8000-000000000000\u0000127\u0000\n";
+ fs.writeFileSync(
+ tool,
+ `printf '%s' "$1"
+/bin/cat <&"$2"
+printf 'synthetic tool failure\\n\\00000000000-0000-4000-8000-000000000000\\000127\\000\\n' >&2
+if [ "$3" = signal ]; then kill -TERM $$; fi
+exit "$3"
+`,
+ { mode: 0o700 },
+ );
+ const original = childProcess.spawn;
+ let failure: NodeJS.ErrnoException = Object.assign(
+ new Error("spawn " + details.code),
+ details,
+ );
+ let attempts = 0;
+ childProcess.spawn = ((
+ ...args: Parameters<typeof childProcess.spawn>
+ ) => {
+ if (args[0] === tool) {
+ attempts++;
+ throw failure;
+ }
+ return original(...args);
+ }) as typeof childProcess.spawn;
+ syncBuiltinESMExports();
+ try {
+ const argument = "spaces 'quoted' $literal\n";
+ const input = Buffer.from([0, 128, 255, 10]);
+ for (const inputFd of [0, 3])
+ for (const status of [0, 126, 127, "signal"]) {
+ const result = await runTool(
+ tool,
+ [argument, String(inputFd), String(status)],
+ root,
+ input,
+ inputFd,
+ );
+ assert.equal(result.status, status === "signal" ? null : status);
+ assert.equal(result.signal, status === "signal" ? "SIGTERM" : null);
+ assert.deepEqual(
+ result.stdout,
+ Buffer.concat([Buffer.from(argument), input]),
+ );
+ assert.equal(result.stderr, diagnostic);
+ }
+ assert.equal(attempts, 8);
+ failure = Object.assign(new Error("unrelated launch failure"), {
+ code: "EACCES",
+ });
+ await assert.rejects(runTool(tool, []), (error) => error === failure);
+ } finally {
+ childProcess.spawn = original;
+ syncBuiltinESMExports();
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+ },
+ );
+
+test(
+ "POSIX tools keep helper input separate from stdin and startup status",
+ { skip: process.platform === "win32" },
+ async () => {
+ const { runTool } = await importSource("src/helpers/inventory-git.ts", {
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ });
+ const root = fs.mkdtempSync(join(tmpdir(), "inventory-input-fds-"));
+ const denied = join(root, "not-executable");
+ fs.writeFileSync(denied, "#!/bin/sh\n", { mode: 0o600 });
+ const unavailableInterpreter = join(root, "unavailable-interpreter");
+ fs.writeFileSync(
+ unavailableInterpreter,
+ "#!/missing-codex-test-interpreter\n",
+ { mode: 0o700 },
+ );
+ try {
+ for (const argument of ["ordinary", "raw-\udcff"])
+ for (const inputFd of [0, 3]) {
+ const input = Buffer.from([0, 128, 255, 10]);
+ const result = await runTool(
+ process.execPath,
+ [
+ "-e",
+ `const fs = require('node:fs');
+fs.writeFileSync(1, fs.readFileSync(${inputFd}));
+${inputFd === 3 ? "fs.writeFileSync(2, String(fs.readFileSync(0).length));" : ""}`,
+ argument,
+ ],
+ undefined,
+ input,
+ inputFd,
+ );
+ assert.equal(result.status, 0, result.stderr);
+ assert.deepEqual(result.stdout, input);
+ assert.equal(result.stderr, inputFd === 3 ? "0" : "");
+ for (const [command, code] of [
+ [join(root, "missing"), "ENOENT"],
+ [unavailableInterpreter, "ENOENT"],
+ [denied, "EACCES"],
+ ])
+ await assert.rejects(
+ runTool(command, [argument], undefined, input, inputFd),
+ { code },
+ );
+ for (const status of [126, 127]) {
+ const launched = await runTool(
+ process.execPath,
+ ["-e", `process.exit(${status})`, argument],
+ undefined,
+ input,
+ inputFd,
+ );
+ assert.equal(launched.status, status);
+ }
+ const statusFd = inputFd === 3 ? 4 : 3;
+ const descendant = await runTool(
+ "/bin/sh",
+ [
+ "-c",
+ `/bin/sh -c 'if ( : >&${statusFd} ) 2>/dev/null; then printf inherited; else printf closed; fi'`,
+ "inventory-fixture",
+ argument,
+ ],
+ undefined,
+ input,
+ inputFd,
+ );
+ assert.equal(descendant.status, 0, descendant.stderr);
+ assert.equal(descendant.stdout.toString(), "closed");
+ }
+ } finally {
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+ },
+);
+
+test(
+ "POSIX tools retain loader settings across the raw argument shell bridge",
+ { skip: process.platform === "win32" },
+ async () => {
+ const { runTool } = await importSource("src/helpers/inventory-git.ts", {
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ });
+ const names = ["LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"];
+ const saved = names.map((name) => [name, process.env[name]] as const);
+ try {
+ for (const name of names) process.env[name] = "/fixture/libraries";
+ for (const argument of ["ordinary", "raw-\udcff"]) {
+ const result = await runTool(process.execPath, [
+ "-e",
+ `process.stdout.write(JSON.stringify(${JSON.stringify(names)}.map(name => process.env[name])));`,
+ argument,
+ ]);
+ assert.equal(result.status, 0, result.stderr);
+ assert.equal(result.stderr, "");
+ assert.deepEqual(JSON.parse(result.stdout.toString()), [
+ "/fixture/libraries",
+ "/fixture/libraries",
+ ]);
+ for (const name of names)
+ assert.equal(process.env[name], "/fixture/libraries");
+ }
+ } finally {
+ for (const [name, value] of saved) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ }
+ },
+);
+
+const { windowsFileSystem } = (await importSource(
+ "../native/windows-files.mts",
+)) as typeof import("../../native/windows-files.mjs");
+
+test(
+ "Windows inventory honors an explicit Git .com executable beside a .com.exe sibling",
+ { skip: process.platform !== "win32" },
+ async () => {
+ const helperUrl = new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ );
+ const { git, runTool } = await importSource(
+ "src/helpers/inventory-git.ts",
+ {
+ define: { "import.meta.url": JSON.stringify(helperUrl.href) },
+ },
+ );
+ const binary = createRequire(helperUrl).resolve(
+ `./native/win32-${process.arch}/windows.node`,
+ );
+ const root = fs.mkdtempSync(join(tmpdir(), "inventory-git-com-"));
+ const repo = join(root, "repository with spaces");
+ const tools = join(root, "trusted tools");
+ const preload = join(root, "probe.cjs");
+ const callerCwd = process.cwd();
+ const names = [
+ "CODEX_SECURITY_GIT",
+ "NODE_OPTIONS",
+ "INVENTORY_PROCESS_FIXTURE",
+ ];
+ const saved = names.map((name) => [name, process.env[name]] as const);
+ try {
+ fs.mkdirSync(repo);
+ fs.mkdirSync(tools);
+ // Preloads run before Node handles Git's -c arguments as syntax checking.
+ fs.writeFileSync(
+ preload,
+ `const fs = require('node:fs');
+const native = require(${JSON.stringify(binary)});
+fs.writeFileSync(1, fs.readFileSync(0));
+fs.writeFileSync(2, JSON.stringify({
+ executable: process.execPath,
+ args: native.windowsArguments().slice(1).map(value => value.toString('utf16le')),
+ cwd: process.cwd(),
+ setting: process.env.INVENTORY_PROCESS_FIXTURE,
+}));
+process.exit(23);
+`,
+ );
+ process.env.NODE_OPTIONS = `--require ${JSON.stringify(preload.replaceAll("\\", "/"))}`;
+ process.env.INVENTORY_PROCESS_FIXTURE = "configured target";
+ for (const basename of ["git.com", ".COM"]) {
+ const executable = join(tools, basename);
+ fs.copyFileSync(process.execPath, executable);
+ fs.copyFileSync(process.execPath, `${executable}.exe`);
+ process.env.CODEX_SECURITY_GIT = executable;
+ for (const argument of ["ordinary", "raw-\ud800\ttab"]) {
+ const args = ["--fixture", argument];
+ const result = await git(repo, args);
+ assert.equal(result.status, 23, result.stderr);
+ assert.equal(result.signal, null);
+ assert.deepEqual(result.stdout, Buffer.alloc(0));
+ assert.deepEqual(JSON.parse(result.stderr), {
+ executable,
+ args: [
+ "-c",
+ "core.fsmonitor=false",
+ "-c",
+ "i18n.logOutputEncoding=UTF-8",
+ "-C",
+ repo,
+ ...args,
+ ],
+ cwd: callerCwd,
+ setting: "configured target",
+ });
+
+ const input = Buffer.from([0, 128, 255, 10]);
+ const toolArgs = ["-e", "", argument];
+ const tool = await runTool(executable, toolArgs, repo, input);
+ assert.equal(tool.status, 23, tool.stderr);
+ assert.equal(tool.signal, null);
+ assert.deepEqual(tool.stdout, input);
+ assert.deepEqual(JSON.parse(tool.stderr), {
+ executable,
+ args: toolArgs,
+ cwd: repo,
+ setting: "configured target",
+ });
+ assert.equal(process.cwd(), callerCwd);
+ }
+ }
+ } finally {
+ for (const [name, value] of saved) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+ },
+);
+
+test(
+ "Windows tools preserve PATH selection and resolve relative PATH and cwd-only executables",
+ { skip: process.platform !== "win32" },
+ async () => {
+ const { runTool } = await importSource("src/helpers/inventory-git.ts", {
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ });
+ const root = fs.mkdtempSync(join(tmpdir(), "inventory-relative-path-"));
+ const repo = join(root, "repo");
+ const tools = join(repo, "tools");
+ const shim = join(root, "shim");
+ const callerCwd = process.cwd();
+ const originalExecutable = process.execPath;
+ const names = [
+ "PATH",
+ "INVENTORY_PROCESS_FIXTURE",
+ "NoDefaultCurrentDirectoryInExePath",
+ ];
+ const saved = names.map((name) => [name, process.env[name]] as const);
+ try {
+ fs.mkdirSync(tools, { recursive: true });
+ fs.copyFileSync(process.execPath, join(tools, "rg.exe"));
+ fs.mkdirSync(shim);
+ fs.copyFileSync(process.execPath, join(shim, "node.exe"));
+ fs.copyFileSync(process.execPath, join(shim, "rg.exe"));
+ process.env.INVENTORY_PROCESS_FIXTURE = "target setting";
+ delete process.env.NoDefaultCurrentDirectoryInExePath;
+ for (const location of ["relative PATH", "PATH collision", "cwd only"]) {
+ process.env.PATH = location === "relative PATH" ? "tools" : tools;
+ process.execPath =
+ location === "PATH collision"
+ ? join(shim, "node.exe")
+ : originalExecutable;
+ if (location === "cwd only")
+ fs.renameSync(join(tools, "rg.exe"), join(repo, "rg.exe"));
+ for (const argument of ["ordinary", "raw-\ud800"]) {
+ const result = await runTool(
+ "rg",
+ [
+ "-e",
+ "process.stdout.write(JSON.stringify({executable:process.execPath,cwd:process.cwd(),setting:process.env.INVENTORY_PROCESS_FIXTURE}));",
+ argument,
+ ],
+ repo,
+ );
+ assert.equal(result.status, 0, result.stderr);
+ assert.deepEqual(JSON.parse(result.stdout.toString()), {
+ executable: join(location === "cwd only" ? repo : tools, "rg.exe"),
+ cwd: repo,
+ setting: "target setting",
+ });
+ assert.equal(result.stderr, "");
+ assert.equal(process.cwd(), callerCwd);
+ }
+ }
+ } finally {
+ process.execPath = originalExecutable;
+ for (const [name, value] of saved) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+ },
+);
+
+for (const platform of ["linux", "win32"]) {
+ test(`Git paths retain platform byte encoding on ${platform}`, async () => {
+ const { decodeGitPath, encodeGitPath } = await importSource(
+ "src/helpers/inventory-git.ts",
+ {
+ define: {
+ "process.platform": JSON.stringify(platform),
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ },
+ );
+ const bytes = Buffer.concat([
+ Buffer.from("résumé/😀/high-"),
+ Buffer.from([0xed, 0xa0, 0x80]),
+ Buffer.from("/low-"),
+ Buffer.from([0xed, 0xbf, 0xbf]),
+ Buffer.from("\0ordinary\0"),
+ ]);
+ const text =
+ platform === "win32"
+ ? "résumé/😀/high-\ud800/low-\udfff\0ordinary\0"
+ : "résumé/😀/high-\udced\udca0\udc80/low-\udced\udcbf\udcbf\0ordinary\0";
+ assert.equal(decodeGitPath(bytes), text);
+ assert.deepEqual(encodeGitPath(text), bytes);
+ });
+}
+
+const { sampleFile, createSourceSampler, PREVIEW_READ_BYTES } =
+ await importSource("src/helpers/source-preview.ts", {
+ define: {
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ });
+
+for (const [name, reparseTag, surrogate] of [
+ ["cloud directory", 0x9000001a, false],
+ ["directory junction", 0xa0000003, true],
+ ["symbolic link", 0xa000000c, true],
+] as const)
+ test(`Windows metadata identifies ${name} without treating every reparse point as a link`, () => {
+ const binding = {
+ windowsAbsolutePath: (value: Buffer) => ({ error: 0, value }),
+ openWindowsFile: () => ({
+ error: 0,
+ handle: {
+ attributes: () => ({ error: 0, attributes: 0x410, reparseTag }),
+ fileType: () => ({ error: 0, value: 1 }),
+ close: () => 0,
+ },
+ }),
+ } as unknown as WindowsBinding;
+ const info = windowsFileSystem(binding).stat(
+ Buffer.from("C:\\fixture", "utf16le"),
+ false,
+ );
+ assert.equal(info.isReparsePoint(), true);
+ assert.equal(info.isNameSurrogate(), surrogate);
+ if (!surrogate) assert.equal(info.isDirectory(), true);
+ });
+
+test("POSIX file identities retain all 64 inode bits", async () => {
+ const original = fs.statSync;
+ fs.statSync = ((path, options) => {
+ if (!["left", "right"].includes(String(path)))
+ return original(path, options);
+ const ino = String(path) === "left" ? 9007199254740992n : 9007199254740993n;
+ return options?.bigint ? { dev: 1n, ino } : { dev: 1, ino: Number(ino) };
+ }) as typeof fs.statSync;
+ syncBuiltinESMExports();
+ try {
+ const { sameFile } = await importSource("src/helpers/inventory-paths.ts", {
+ define: {
+ "process.platform": '"linux"',
+ "import.meta.url": JSON.stringify(
+ new URL(
+ "../../../../sdk/typescript/_bundled_plugin/mcp/helpers.mjs",
+ import.meta.url,
+ ).href,
+ ),
+ },
+ });
+ assert.equal(sameFile("left", "right"), false);
+ assert.equal(sameFile("left", "left"), true);
+ } finally {
+ fs.statSync = original;
+ syncBuiltinESMExports();
+ }
+});
+
+test(
+ "short file reads preserve the complete preview prefix and UTF-16 units",
+ { skip: process.platform === "win32" },
+ () => {
+ const root = fs.mkdtempSync(join(tmpdir(), "inventory-short-read-"));
+ const file = join(root, "source");
+ const original = fs.readSync;
+ let calls = 0;
+ fs.readSync = ((descriptor, buffer, offset, length, position) =>
+ original(
+ descriptor,
+ buffer,
+ offset,
+ Math.min(length, [1, 3, 5, 4096][calls++ % 4]!),
+ position,
+ )) as typeof fs.readSync;
+ syncBuiltinESMExports();
+ try {
+ const utf16 = Buffer.concat([
+ Buffer.from([0xff, 0xfe]),
+ Buffer.from("Ā source\n".repeat(8000), "utf16le"),
+ ]);
+ for (const data of [
+ Buffer.alloc(0),
+ Buffer.from([0xff]),
+ Buffer.from([0xff, 0xfe]),
+ Buffer.from([0xff, 0xfe, 0]),
+ Buffer.from([0xfe, 0xff, 0]),
+ Buffer.from("source\n".repeat(12000)),
+ utf16,
+ Buffer.from(utf16).swap16(),
+ ]) {
+ fs.writeFileSync(file, data);
+ calls = 0;
+ const [sample, binary] = sampleFile(file);
+ assert.equal(binary, false);
+ assert.deepEqual(sample, data.subarray(0, PREVIEW_READ_BYTES));
+ fs.writeFileSync(file, Buffer.concat([data, Buffer.from([0, 0])]));
+ calls = 0;
+ assert.equal(sampleFile(file)[1], true);
+ }
+ } finally {
+ fs.readSync = original;
+ syncBuiltinESMExports();
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+ },
+);
+
+test("Windows short file reads preserve previews across split BOMs and units", () => {
+ const text = Buffer.concat([
+ Buffer.from([0xff, 0xfe]),
+ Buffer.from("Ā source\n".repeat(8000), "utf16le"),
+ ]);
+ for (const source of [Buffer.alloc(0), Buffer.from([0xff, 0xfe, 0]), text])
+ for (const binary of [false, true]) {
+ const data = binary
+ ? Buffer.concat([source, Buffer.from([0, 0])])
+ : source;
+ let cursor = 0,
+ calls = 0,
+ closes = 0;
+ const native = {
+ windowsAbsolutePath: (value: Buffer) => ({ error: 0, value }),
+ openWindowsFile: () => ({
+ error: 0,
+ handle: {
+ read: (buffer: Buffer, offset: number, length: number) => {
+ const count = Math.min(
+ length,
+ data.length - cursor,
+ [1, 3, 5, 4096][calls++ % 4]!,
+ );
+ data.copy(buffer, offset, cursor, cursor + count);
+ cursor += count;
+ return { error: 0, value: count };
+ },
+ close: () => {
+ closes++;
+ return 0;
+ },
+ },
+ }),
+ } as unknown as WindowsBinding;
+ const sampler = createSourceSampler();
+ assert.equal(sampler.consume(Buffer.alloc(0)), true);
+ windowsFileSystem(native).readChunks(
+ Buffer.from("C:\\fixture", "utf16le"),
+ sampler.consume,
+ );
+ assert.deepEqual(sampler.finish(), [
+ binary ? Buffer.alloc(0) : source.subarray(0, PREVIEW_READ_BYTES),
+ binary,
+ ]);
+ assert.equal(closes, 1);
+ }
+});
diff --git a/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts b/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts
index 3a6ed7e4f..ca3e596fd 100644
--- a/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts
+++ b/plugins/codex-security/mcp-app/tests/test_mcp_app_smoke.ts
@@ -475,8 +475,9 @@ async function assertWorkspaceWorksWithoutUiCapability(scope: string) {
);
}
const inventoryPath = path.join(fixtureRoot, "in_scope_files.txt");
- execFileSync(process.env.PYTHON?.trim() || "python3", [
- path.join(pluginRoot, "scripts", "generate_in_scope_files.py"),
+ execFileSync(process.execPath, [
+ path.join(pluginRoot, "mcp", "helpers.mjs"),
+ "generate-in-scope-files",
"--repo",
canonicalTarget,
`--scope=${scope}`,
diff --git a/plugins/codex-security/native/examples/windows-wide-launcher.rs b/plugins/codex-security/native/examples/windows-wide-launcher.rs
index 65be54713..2a1ee5944 100644
--- a/plugins/codex-security/native/examples/windows-wide-launcher.rs
+++ b/plugins/codex-security/native/examples/windows-wide-launcher.rs
@@ -230,6 +230,40 @@ fn main() -> std::io::Result<()> {
));
}
}
+ // Keep the Node 20 ABI proof independent of its win32.relative bug for
+ // case-folding expansions such as İ. The SDK tests inventory on supported Node versions.
+ let inventory_repo = root.join(&cwds[0]).join(raw("inventory-", 0xdc80));
+ let inventory_scope = inventory_repo.join(&scopes[0]);
+ fs::create_dir_all(&inventory_scope)?;
+ fs::write(inventory_scope.join("SECURITY.md"), "scope raw\n")?;
+ let inventory_output = inventory_repo.join(&output_name);
+ let inventory_scopes = inventory_repo.join("inventory-scopes.json");
+ fs::write(&inventory_scopes, r#"["scope-\udfff"]"#)?;
+ let expected = "{\"path\":\"scope-\\udfff/SECURITY.md\"}\r\n";
+ let generated = invoke(
+ "make-repo-scope-input",
+ &[
+ "--repo".into(),
+ inventory_repo.clone(),
+ "--scopes-file".into(),
+ inventory_scopes.clone(),
+ "--out".into(),
+ inventory_output.clone(),
+ ],
+ )?;
+ let actual = fs::read(&inventory_output);
+ if !generated.status.success()
+ || !generated.stderr.is_empty()
+ || !actual
+ .as_ref()
+ .is_ok_and(|bytes| bytes == expected.as_bytes())
+ {
+ return Err(io::Error::other(format!(
+ "Wide scoped inventory helper failed: status={}, stdout={:?}, stderr={:?}, expected={:?}, actual={actual:?}",
+ generated.status, generated.stdout, generated.stderr, expected.as_bytes(),
+ )));
+ }
+ fs::remove_dir_all(inventory_repo)?;
let identity_root = root.join("İroot");
let sibling = root.join("i\u{307}root");
fs::create_dir(&identity_root)?;
@@ -511,7 +545,7 @@ fn main() -> std::io::Result<()> {
}
}
println!(
- "{{\"policyHelperRawPaths\":true,\"candidateHelperRawPaths\":true,\"assessmentHelperRawPaths\":true,\"deepReviewHelperRawPaths\":true,\"rankShardHelperRawPaths\":true,\"rankPoolHelperRawPaths\":true,\"bindScopesHelperRawPaths\":true,\"directoryIdentity\":true,\"policySymlinkBoundary\":{symlinks}}}"
+ "{{\"policyHelperRawPaths\":true,\"candidateHelperRawPaths\":true,\"assessmentHelperRawPaths\":true,\"deepReviewHelperRawPaths\":true,\"rankShardHelperRawPaths\":true,\"rankPoolHelperRawPaths\":true,\"bindScopesHelperRawPaths\":true,\"inventoryHelperRawPaths\":true,\"directoryIdentity\":true,\"policySymlinkBoundary\":{symlinks}}}"
);
Ok(())
}
diff --git a/plugins/codex-security/native/proof-windows.mts b/plugins/codex-security/native/proof-windows.mts
index d0399c7c4..62aef461f 100644
--- a/plugins/codex-security/native/proof-windows.mts
+++ b/plugins/codex-security/native/proof-windows.mts
@@ -251,10 +251,12 @@ function handleProof(root: string) {
const junctionStat = files.stat(pathBytes(ancestor), false);
assert(junctionStat.isDirectory());
assert(junctionStat.isReparsePoint());
+ assert(junctionStat.isNameSurrogate());
assert(!junctionStat.isSymbolicLink());
const targetStat = files.stat(pathBytes(ancestor));
assert(targetStat.isDirectory());
assert(!targetStat.isReparsePoint());
+ assert(!targetStat.isNameSurrogate());
samePath(
checked(junction.finalPath(flags.FILE_NAME_OPENED)).path,
ancestor,
diff --git a/plugins/codex-security/native/windows-files.mts b/plugins/codex-security/native/windows-files.mts
index e11091fe9..b360b2841 100644
--- a/plugins/codex-security/native/windows-files.mts
+++ b/plugins/codex-security/native/windows-files.mts
@@ -115,6 +115,7 @@ export function windowsFileSystem(native: WindowsBinding) {
isSymbolicLink: () => link,
isReparsePoint: () =>
(info.attributes & flags.FILE_ATTRIBUTE_REPARSE_POINT) !== 0,
+ isNameSurrogate: () => (info.reparseTag & 0x20000000) !== 0,
};
},
flags.OPEN_EXISTING,
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json
index 748822711..65986f053 100644
--- a/plugins/codex-security/plugin-files.json
+++ b/plugins/codex-security/plugin-files.json
@@ -13,6 +13,7 @@
"examples/completed-scan/scan-manifest.json",
"mcp/helpers.mjs",
"mcp/helpers.mjs.br.part-000",
+ "mcp/helpers.mjs.br.part-001",
"mcp/native/COPYRIGHT-library.html",
"mcp/native/THIRD_PARTY_NOTICES.txt",
"mcp/native/darwin-arm64/unix.node",
@@ -67,11 +68,8 @@
"scripts/filesystem_identity.py",
"scripts/finalize_scan_contract.py",
"scripts/finding_preview.py",
- "scripts/generate_in_scope_files.py",
- "scripts/generate_rank_input.py",
"scripts/launch_codex_security_mcp",
"scripts/launch_codex_security_mcp.cmd",
- "scripts/rank_preview.py",
"scripts/report_projection.py",
"scripts/reserved_artifact_paths.json",
"scripts/snapshot_sqlite.py",
diff --git a/plugins/codex-security/scripts/generate_in_scope_files.py b/plugins/codex-security/scripts/generate_in_scope_files.py
deleted file mode 100644
index cdcacde9a..000000000
--- a/plugins/codex-security/scripts/generate_in_scope_files.py
+++ /dev/null
@@ -1,277 +0,0 @@
-#!/usr/bin/env python3
-"""Generate the shared, deterministically ordered security-scan file inventory."""
-
-from __future__ import annotations
-
-import argparse
-import os
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-# Some plugin hosts launch Python with safe-path isolation enabled.
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from workbench_target import SOURCE_METADATA_DIRECTORIES, git_blob_samples, git_command
-
-
-class InventoryError(ValueError):
- """Raised when the repository, scope, or inventory cannot be used safely."""
-
-
-def windows_stream_component(path: Path) -> str | None:
- """Return the first NTFS alternate-data-stream component."""
-
- if os.name != "nt":
- return None
- return next(
- (component for component in path.parts if component != path.anchor and ":" in component),
- None,
- )
-
-
-def resolve_repository(value: str) -> Path:
- """Resolve the repository once so every scope is bound to its real root."""
- try:
- repository = Path(value).expanduser().resolve(strict=True)
- except (OSError, ValueError) as error:
- raise InventoryError(f"--repo: cannot resolve repository: {value}") from error
- if not repository.is_dir():
- raise InventoryError(f"--repo: expected a directory: {repository}")
- return repository
-
-
-def resolve_scope(repository: Path, value: str) -> str:
- """Preserve ripgrep's relative path spelling while rejecting escaped scopes."""
- if not value or "\0" in value:
- raise InventoryError("--scope: expected a non-empty file or directory")
-
- requested = Path(value).expanduser()
- stream = windows_stream_component(requested)
- if stream is not None:
- raise InventoryError(f"--scope: NTFS alternate data streams are not supported: {stream}")
- scope = requested if requested.is_absolute() else repository / requested
- try:
- resolved = scope.resolve(strict=True)
- except (OSError, ValueError) as error:
- raise InventoryError(f"--scope: path does not exist: {value}") from error
-
- try:
- relative = resolved.relative_to(repository)
- except ValueError as error:
- raise InventoryError(f"--scope: path must remain inside --repo: {value}") from error
-
- if not resolved.is_dir() and not resolved.is_file():
- raise InventoryError(f"--scope: expected a file or directory: {value}")
-
- if requested.is_absolute():
- return relative.as_posix() if relative.parts else "."
- return value
-
-
-def resolve_output(value: str) -> Path:
- """Reject direct symlink outputs without constraining the artifact root."""
- if not value or "\0" in value:
- raise InventoryError("--out: expected an inventory file path")
- requested = Path(value).expanduser()
- if requested.is_symlink():
- raise InventoryError("--out: refusing to replace a symbolic link")
- try:
- output = requested.resolve(strict=False)
- except (OSError, ValueError) as error:
- raise InventoryError(f"--out: cannot resolve inventory path: {value}") from error
- if output.exists() and not output.is_file():
- raise InventoryError(f"--out: expected a regular file path: {output}")
- return output
-
-
-def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
- """Atomically write the exact ripgrep inventory sorted as ``LC_ALL=C``."""
- command = [
- "rg",
- "--files",
- "--null",
- "--hidden",
- "--path-separator",
- "/",
- # Prune metadata and its contents even when the scope starts inside it.
- "--glob",
- "!**/{.git,.svn}",
- "--glob",
- "!**/{.git,.svn}/**",
- "--",
- scope,
- ]
- with tempfile.TemporaryFile(mode="w+b") as inventory:
- try:
- result = subprocess.run(
- command,
- cwd=repository,
- stdout=inventory,
- stderr=subprocess.PIPE,
- check=False,
- )
- except OSError as error:
- raise InventoryError(f"could not run ripgrep: {error}") from error
-
- if result.returncode not in (0, 1):
- detail = result.stderr.decode("utf-8", errors="replace").strip()
- message = f"ripgrep exited with status {result.returncode}"
- if detail:
- message = f"{message}: {detail}"
- raise InventoryError(message)
-
- if any((parent / ".git").exists() for parent in (repository, *repository.parents)):
- try:
- tracked = git_command(
- repository,
- "ls-files",
- "--cached",
- "--ignored",
- "--exclude-standard",
- "-z",
- "--",
- scope,
- text=False,
- )
- except OSError:
- tracked = None
- if tracked is not None and tracked.returncode == 0:
- prefix = b"./" if scope == "." or scope.startswith("./") else b""
- for path in tracked.stdout.split(b"\0"):
- candidate = repository / os.fsdecode(path)
- if path and candidate.is_file() and not candidate.is_symlink():
- inventory.write(prefix + path + b"\0")
-
- inventory.seek(0)
- rows: list[bytes] = []
- for path in inventory.read().split(b"\0"):
- if not path or not SOURCE_METADATA_DIRECTORIES.isdisjoint(os.fsdecode(path).split("/")):
- continue
- if b"\n" in path or b"\r" in path:
- raise InventoryError(
- "Repository contains a path that cannot fit in the file inventory"
- )
- rows.append(path + b"\n")
-
- return write_inventory(output, sorted(set(rows)))
-
-
-def generate_diff_in_scope_files(
- repository: Path,
- base: str,
- head: str,
- mode: str,
- output: Path,
-) -> int:
- """Reuse the existing diff selection without generating previews or duplicate worklists."""
- from generate_rank_input import git_changed_paths
- from rank_preview import is_binary_file
-
- rows: list[bytes] = []
- try:
- changed = git_changed_paths(repository, base, head, mode)
- revision_refs = {
- path.relative_to(repository): (
- f"{base if status == 'D' else head}:{path.relative_to(repository).as_posix()}"
- )
- for path, status in changed
- if mode == "revisions" or status == "D"
- }
- revision_samples = dict(
- zip(revision_refs, git_blob_samples(repository, list(revision_refs.values())))
- )
-
- for path, status in changed:
- relative = path.relative_to(repository)
- if mode == "revisions" or status == "D":
- sample = revision_samples[relative]
- if sample is None:
- revision = base if status == "D" else head
- raise InventoryError(
- f"could not read committed diff blob: {revision}:{relative.as_posix()}"
- )
- _, is_binary = sample
- if is_binary:
- continue
- elif path.is_symlink() or not path.is_file() or is_binary_file(path):
- continue
- relative_path = os.fsencode(relative.as_posix())
- if b"\n" in relative_path or b"\r" in relative_path:
- raise InventoryError(
- "Git changes contain a path that cannot fit in the file inventory"
- )
- rows.append(relative_path + b"\n")
- except (OSError, subprocess.CalledProcessError) as error:
- detail = getattr(error, "stderr", None)
- if isinstance(detail, bytes):
- detail = detail.decode("utf-8", errors="replace")
- message = detail.strip() if isinstance(detail, str) and detail.strip() else str(error)
- raise InventoryError(f"could not resolve the selected Git changes: {message}") from error
-
- return write_inventory(output, sorted(set(rows)))
-
-
-def write_inventory(output: Path, rows: list[bytes]) -> int:
- """Replace a complete inventory atomically, keeping failures from corrupting the old one."""
- output.parent.mkdir(parents=True, exist_ok=True)
- temporary: Path | None = None
- try:
- with tempfile.NamedTemporaryFile(
- mode="wb",
- dir=output.parent,
- # Including the output name can exceed the filesystem's 255-byte name limit.
- prefix=".",
- suffix=".tmp",
- delete=False,
- ) as handle:
- temporary = Path(handle.name)
- handle.writelines(rows)
- temporary.replace(output)
- finally:
- if temporary is not None:
- temporary.unlink(missing_ok=True)
-
- return len(rows)
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--repo", required=True, help="Repository root.")
- parser.add_argument("--scope", required=True, help="File or directory within the repository.")
- parser.add_argument("--out", required=True, help="Destination for the file inventory.")
- parser.add_argument("--diff-base", help="Authoritative Git base for a changed-file inventory.")
- parser.add_argument("--diff-head", default="HEAD", help="Authoritative Git head revision.")
- parser.add_argument(
- "--diff-mode",
- choices=("revisions", "local-patch"),
- default="revisions",
- help="Use committed revisions or the current staged and unstaged patch.",
- )
- args = parser.parse_args()
-
- try:
- repository = resolve_repository(args.repo)
- scope = resolve_scope(repository, args.scope)
- output = resolve_output(args.out)
- if args.diff_base is None:
- count = generate_in_scope_files(repository, scope, output)
- elif scope not in (".", "./"):
- raise InventoryError("--scope: diff scans must use the repository root")
- else:
- count = generate_diff_in_scope_files(
- repository,
- args.diff_base,
- args.diff_head,
- args.diff_mode,
- output,
- )
- except (OSError, ValueError) as error:
- print(f"generate_in_scope_files: {error}", file=sys.stderr)
- raise SystemExit(2) from error
-
- print(f"Recorded {count} in-scope files.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/plugins/codex-security/scripts/generate_rank_input.py b/plugins/codex-security/scripts/generate_rank_input.py
deleted file mode 100644
index ae7f90e5c..000000000
--- a/plugins/codex-security/scripts/generate_rank_input.py
+++ /dev/null
@@ -1,350 +0,0 @@
-#!/usr/bin/env python3
-"""Generate and post-process Codex Security scan worklists.
-
-This script stays deliberately model-free:
-
-- `make-repo-scope-input` lists every explicitly scoped repository file.
-- `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
- worklist from Git changed paths. It supports committed revision diffs and
- local working-tree patches.
-
-Candidate selection uses repository scope, native tool ignore rules, and binary detection,
-not filename or directory classifications. Diff inputs use bounded source previews.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import subprocess
-import sys
-from collections.abc import Iterable
-from pathlib import Path
-
-# Some plugin hosts launch Python with safe-path isolation enabled.
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-from generate_in_scope_files import windows_stream_component
-from rank_preview import (
- DEFAULT_PREVIEW_BYTES,
- is_binary_file,
- preview_for,
- preview_for_bytes,
-)
-from workbench_target import (
- SOURCE_METADATA_DIRECTORIES,
- git_blob_samples,
- git_command,
- git_directory_snapshot_paths,
-)
-
-JsonRow = dict[str, object]
-
-
-def parse_args() -> argparse.Namespace:
- parser = argparse.ArgumentParser(description="Codex Security scan worklist helper.")
- subparsers = parser.add_subparsers(dest="command", required=True)
-
- scoped = subparsers.add_parser(
- "make-repo-scope-input",
- help="List every explicitly scoped file without ranking or reading its contents.",
- )
- scoped.add_argument("--repo", required=True, help="Repository root.")
- scoped.add_argument(
- "--scopes-file",
- required=True,
- help="JSON array of repository-relative files and directories to scan together.",
- )
- scoped.add_argument("--out", required=True, help="Output scoped-source-input.jsonl path.")
-
- diff = subparsers.add_parser(
- "make-diff-rank-input",
- help="Create rank_input.jsonl from Git changed text files.",
- )
- diff.add_argument("--repo", required=True, help="Repository root.")
- diff.add_argument("--base", required=True, help="Git diff base revision.")
- diff.add_argument(
- "--mode",
- choices=("revisions", "local-patch"),
- default="revisions",
- help="Git diff mode: committed revisions or staged plus unstaged local patch.",
- )
- diff.add_argument("--head", default="HEAD", help="Git diff head revision.")
- diff.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
- diff.add_argument("--area", default="diff", help="Area label for changed-file rows.")
- diff.add_argument(
- "--preview-bytes",
- type=int,
- default=DEFAULT_PREVIEW_BYTES,
- help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
- )
-
- return parser.parse_args()
-
-
-def resolve_scope(repo: Path, scope: str) -> Path:
- scope_path = Path(scope)
- stream = windows_stream_component(scope_path)
- if stream is not None:
- raise SystemExit(f"Scope must not use an NTFS alternate data stream: {stream}")
- if not scope_path.is_absolute():
- scope_path = repo / scope_path
- repository = repo.resolve()
- try:
- relative = scope_path.relative_to(repository)
- except ValueError as exc:
- raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
- ancestor = repository
- for part in relative.parts:
- if part == "..":
- if ancestor == repository:
- raise SystemExit(f"Scope must be inside repo: {scope_path}")
- ancestor = ancestor.parent
- continue
- ancestor /= part
- try:
- metadata = ancestor.stat(follow_symlinks=False)
- except OSError as exc:
- raise SystemExit(f"Scope path not found: {ancestor}") from exc
- if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
- raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")
- scope_path = scope_path.resolve()
- repo_resolved = repo.resolve()
- try:
- scope_path.relative_to(repo_resolved)
- except ValueError as exc:
- raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
- if not scope_path.is_dir() and not scope_path.is_file():
- raise SystemExit(f"Scope path not found: {scope_path}")
- return scope_path
-
-
-def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
- output.parent.mkdir(parents=True, exist_ok=True)
- with output.open("w", encoding="utf-8") as handle:
- for row in rows:
- handle.write(json.dumps(row, ensure_ascii=True, separators=(",", ":")))
- handle.write("\n")
-
-
-def load_scopes_file(scopes_file: Path) -> list[str]:
- try:
- loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
- except (OSError, UnicodeError, json.JSONDecodeError) as exc:
- raise SystemExit(f"Unable to read scopes file: {scopes_file}") from exc
- if (
- not isinstance(loaded, list)
- or not loaded
- or any(not isinstance(scope, str) or not scope for scope in loaded)
- ):
- raise SystemExit(f"Scopes file must contain a non-empty JSON string array: {scopes_file}")
- return loaded
-
-
-def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
- """Use Git's inventory in worktrees and ripgrep's ignore rules elsewhere.
-
- Git retains tracked files and applies its standard exclusions to untracked files.
- Outside Git, ripgrep also honors .ignore and .rgignore alongside .gitignore.
- Explicit file scopes bypass directory ignore rules.
- """
- if scope_path.is_file():
- return (scope_path,)
- git_candidates = git_directory_snapshot_paths(scope_path)
- if git_candidates is not None:
- return git_candidates
-
- command = [
- "rg",
- "--files",
- "--hidden",
- "--no-require-git",
- "--null",
- # Also exclude descendants when the scope starts inside metadata.
- "--glob",
- "!**/{.git,.svn}",
- "--glob",
- "!**/{.git,.svn}/**",
- "--",
- str(scope_path.relative_to(repo)),
- ]
- try:
- result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
- except OSError as exc:
- ignore_names = (".gitignore", ".ignore", ".rgignore")
- ancestors = (scope_path, *scope_path.parents)
- has_ignore_rules = (
- any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
- or any(
- (ancestor / name).is_file()
- for ancestor in ancestors
- if ancestor == repo or repo in ancestor.parents
- for name in ignore_names
- )
- or any(path.name in ignore_names for path in scope_path.rglob("*") if path.is_file())
- )
- if has_ignore_rules:
- raise SystemExit(
- "Could not safely enumerate ignored scoped files without Git or ripgrep."
- ) from exc
- return scope_path.rglob("*")
-
- if result.returncode not in (0, 1):
- detail = result.stderr.decode("utf-8", errors="replace").strip()
- raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
- return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
-
-
-def make_repo_scope_input(args: argparse.Namespace) -> None:
- repo = Path(args.repo).expanduser().resolve()
- if not repo.is_dir():
- raise SystemExit(f"Repo path not found: {repo}")
-
- scopes = load_scopes_file(Path(args.scopes_file).expanduser())
- rows_by_path: dict[str, JsonRow] = {}
- for scope in scopes:
- scope_path = resolve_scope(repo, scope)
- candidates = scope_candidates(repo, scope_path)
- for path in candidates:
- try:
- if path.is_symlink() or not path.is_file():
- continue
- relative = path.resolve(strict=True).relative_to(repo)
- except (OSError, ValueError):
- continue
- if not SOURCE_METADATA_DIRECTORIES.isdisjoint(relative.parts):
- continue
- rows_by_path.setdefault(relative.as_posix(), {"path": relative.as_posix()})
-
- rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"]))
- output = Path(args.out).expanduser()
- write_jsonl(output, rows)
- print(f"Wrote {len(rows)} scoped paths to {output}")
-
-
-def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
- """Return changed regular files from the selected side of each change."""
- result = git_command(
- repo,
- "diff",
- "--ignore-submodules=all",
- "--raw",
- "-z",
- "--diff-filter=ACMRDT",
- *diff_args,
- "--",
- text=False,
- )
- result.check_returncode()
- fields = result.stdout.split(b"\0")
- if fields and not fields[-1]:
- fields.pop()
-
- changed: list[tuple[Path, str]] = []
- index = 0
- while index < len(fields):
- metadata = fields[index].split()
- status = chr(metadata[-1][0])
- index += 1
- if status in {"C", "R"}:
- index += 1
- path = repo / os.fsdecode(fields[index])
- index += 1
- selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
- if selected_mode.startswith(b"100"):
- changed.append((path, status))
- return changed
-
-
-def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:
- if mode == "revisions":
- return run_git_changed_paths(repo, [f"{base}..{head}"])
- if mode == "local-patch":
- unstaged = run_git_changed_paths(repo, [base])
- staged = run_git_changed_paths(repo, ["--cached", base])
- untracked = git_command(
- repo,
- "ls-files",
- "--others",
- "--exclude-standard",
- "-z",
- text=False,
- )
- untracked.check_returncode()
- combined = dict(staged)
- combined.update(unstaged)
- combined.update(
- (repo / os.fsdecode(relative), "A")
- for relative in untracked.stdout.split(b"\0")
- if relative and not relative.endswith(b"/")
- )
- return sorted(
- (path, status)
- for path, status in combined.items()
- if SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts)
- and (status == "D" or (not path.is_symlink() and path.is_file()))
- )
- raise SystemExit(f"Unknown diff mode: {mode}")
-
-
-def make_diff_rank_input(args: argparse.Namespace) -> None:
- repo = Path(args.repo).expanduser().resolve()
- if not repo.is_dir():
- raise SystemExit(f"Repo path not found: {repo}")
-
- changed = git_changed_paths(repo, args.base, args.head, args.mode)
- revision_refs = {
- path.relative_to(repo): (
- f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
- )
- for path, status in changed
- if args.mode == "revisions" or status == "D"
- }
- revision_samples = dict(
- zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
- )
-
- rows: list[JsonRow] = []
- for path, status in changed:
- rel = path.relative_to(repo)
-
- preview = ""
- if args.mode == "revisions" or status == "D":
- sample = revision_samples[rel]
- if sample is None:
- revision = args.base if status == "D" else args.head
- raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
- content, is_binary = sample
- if is_binary:
- continue
- if status != "D":
- preview, _ = preview_for_bytes(content, args.preview_bytes)
- elif not path.is_symlink() and path.is_file():
- try:
- path.resolve(strict=True).relative_to(repo)
- except (OSError, ValueError):
- preview = ""
- else:
- preview, is_binary = preview_for(path, args.preview_bytes)
- if is_binary or is_binary_file(path):
- continue
- rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
-
- rows.sort(key=lambda row: str(row["path"]))
- output = Path(args.out).expanduser()
- write_jsonl(output, rows)
- print(f"Wrote {len(rows)} rows to {output}")
-
-
-def main() -> None:
- args = parse_args()
- if args.command == "make-repo-scope-input":
- make_repo_scope_input(args)
- elif args.command == "make-diff-rank-input":
- make_diff_rank_input(args)
- else:
- raise SystemExit(f"Unknown command: {args.command}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.py
deleted file mode 100644
index f095484bc..000000000
--- a/plugins/codex-security/scripts/rank_preview.py
+++ /dev/null
@@ -1,129 +0,0 @@
-"""Build bounded source previews for rank inputs regardless of language."""
-
-from __future__ import annotations
-
-import argparse
-from bisect import bisect_right
-from pathlib import Path
-
-DEFAULT_PREVIEW_BYTES = 1024
-DEFAULT_PREVIEW_READ_BYTES = 64 * 1024
-PREVIEW_HEAD_LINES = 12
-PREVIEW_SAMPLE_LINES = 10
-_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
-
-
-def _decode_source(data: bytes) -> str:
- encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8-sig"
- return data.decode(encoding, errors="ignore")
-
-
-def is_binary_sample(data: bytes) -> bool:
- if data.startswith(_UTF16_BOMS):
- return "\0" in _decode_source(data)
- return b"\0" in data
-
-
-def is_binary_file(path: Path) -> bool:
- """Classify the whole file in bounded chunks."""
- try:
- with path.open("rb") as source:
- sample = source.read(DEFAULT_PREVIEW_READ_BYTES)
- if is_binary_sample(sample):
- return True
- bom = sample[:2] if sample.startswith(_UTF16_BOMS) else b""
- while chunk := source.read(DEFAULT_PREVIEW_READ_BYTES):
- # Even-sized reads preserve UTF-16 code-unit alignment and byte order.
- if is_binary_sample(bom + chunk) if bom else b"\0" in chunk:
- return True
- except OSError:
- return True
- return False
-
-
-def truncate_utf8(text: str, max_bytes: int) -> str:
- if max_bytes <= 0:
- return ""
- encoded = text.encode("utf-8")
- if len(encoded) <= max_bytes:
- return text
- return encoded[:max_bytes].decode("utf-8", errors="ignore")
-
-
-def select_preview_lines(lines: list[str]) -> list[str]:
- nonblank = [line for line in lines if line.strip()]
- head = nonblank[:PREVIEW_HEAD_LINES]
- remainder = nonblank[PREVIEW_HEAD_LINES:]
- if len(remainder) <= PREVIEW_SAMPLE_LINES:
- return [*head, *remainder]
-
- last_index = len(remainder) - 1
- sampled = [
- remainder[index * last_index // (PREVIEW_SAMPLE_LINES - 1)]
- for index in range(PREVIEW_SAMPLE_LINES)
- ]
- return [*head, "...", *sampled]
-
-
-def fit_preview_lines(lines: list[str], max_bytes: int) -> str:
- if not lines or max_bytes <= 0:
- return ""
-
- full_preview = "\n".join(lines)
- if len(full_preview.encode("utf-8")) <= max_bytes:
- return full_preview
-
- def render(line_bytes: int) -> str:
- return "\n".join(
- line if line == "..." else truncate_utf8(line, line_bytes) for line in lines
- )
-
- high = max(len(line.encode("utf-8")) for line in lines)
- limit = bisect_right(
- range(high + 1), max_bytes, key=lambda size: len(render(size).encode("utf-8"))
- )
- best = render(limit - 1) if limit else ""
- if best.strip() not in {"", "..."}:
- return best
- return truncate_utf8(full_preview, max_bytes)
-
-
-def preview_for(
- path: Path, preview_bytes: int, *, max_read_bytes: int = DEFAULT_PREVIEW_READ_BYTES
-) -> tuple[str, bool]:
- """Preview the first 64 KiB by default, including the binary-detection sample."""
- if max_read_bytes <= 0:
- raise ValueError("max_read_bytes must be positive")
- try:
- with path.open("rb") as source:
- sample = source.read(min(4096, max_read_bytes))
- if is_binary_sample(sample):
- return "", True
- remaining = source.read(max_read_bytes - len(sample))
- data = sample + remaining
- except OSError:
- return "", True
- return preview_for_bytes(data, preview_bytes)
-
-
-def preview_for_bytes(data: bytes, preview_bytes: int) -> tuple[str, bool]:
- if is_binary_sample(data):
- return "", True
- lines = _decode_source(data).replace("\r\n", "\n").replace("\r", "\n").split("\n")
- start, end = 0, len(lines)
- while start < end and not lines[start].strip():
- start += 1
- while end > start and not lines[end - 1].strip():
- end -= 1
- lines = lines[start:end]
- text = "\n".join(lines)
- if len(text.encode("utf-8")) <= preview_bytes:
- return text, False
- return fit_preview_lines(select_preview_lines(lines), preview_bytes), False
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(
- description="Build bounded source previews for Codex Security rank inputs."
- )
- parser.parse_args()
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index 603455ff0..ecff064fb 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -20,7 +20,6 @@ from typing import Any, BinaryIO, Callable
# Some plugin hosts launch Python with safe-path isolation enabled.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from filesystem_identity import stored_filesystem_identity_matches
-from rank_preview import DEFAULT_PREVIEW_READ_BYTES, is_binary_sample
from windows_scan_local_files import copy_directory_junction
from workbench_constants import GIT_REPOSITORY_ENVIRONMENT
@@ -162,22 +161,29 @@ def _read_git_batch_samples(
return samples
+def _is_binary_sample(data: bytes) -> bool:
+ if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+ return "\0" in data.decode("utf-16", errors="ignore")
+ return b"\0" in data
+
+
def _read_git_blob_sample(output: BinaryIO, size: int) -> tuple[bytes, bool]:
- sample_size = min(size, DEFAULT_PREVIEW_READ_BYTES)
+ read_size = 64 * 1024
+ sample_size = min(size, read_size)
sample = output.read(sample_size)
if len(sample) != sample_size:
raise ValueError("truncated blob")
- binary = is_binary_sample(sample)
+ binary = _is_binary_sample(sample)
bom = sample[:2] if sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)) else b""
remaining = size - sample_size
while remaining:
- chunk_size = min(remaining, DEFAULT_PREVIEW_READ_BYTES)
+ chunk_size = min(remaining, read_size)
chunk = output.read(chunk_size)
if len(chunk) != chunk_size:
raise ValueError("truncated blob")
# Even-sized reads keep UTF-16 code units aligned; preserve the original byte order.
if not binary:
- binary = is_binary_sample(bom + chunk) if bom else b"\0" in chunk
+ binary = _is_binary_sample(bom + chunk) if bom else b"\0" in chunk
remaining -= chunk_size
if output.read(1) != b"\n":
raise ValueError("missing blob terminator")
@@ -310,6 +316,71 @@ def git_command(
return subprocess.CompletedProcess(full_command, 127, empty_output, empty_output)
+def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
+ """Return changed regular files from the selected side of each change."""
+ result = git_command(
+ repo,
+ "diff",
+ "--ignore-submodules=all",
+ "--raw",
+ "-z",
+ "--diff-filter=ACMRDT",
+ *diff_args,
+ "--",
+ text=False,
+ )
+ result.check_returncode()
+ fields = result.stdout.split(b"\0")
+ if fields and not fields[-1]:
+ fields.pop()
+
+ changed: list[tuple[Path, str]] = []
+ index = 0
+ while index < len(fields):
+ metadata = fields[index].split()
+ status = chr(metadata[-1][0])
+ index += 1
+ if status in {"C", "R"}:
+ index += 1
+ path = repo / os.fsdecode(fields[index])
+ index += 1
+ selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
+ if selected_mode.startswith(b"100"):
+ changed.append((path, status))
+ return changed
+
+
+def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:
+ if mode == "revisions":
+ return run_git_changed_paths(repo, [f"{base}..{head}"])
+ if mode == "local-patch":
+ unstaged = run_git_changed_paths(repo, [base])
+ staged = run_git_changed_paths(repo, ["--cached", base])
+ untracked = git_command(
+ repo,
+ "ls-files",
+ "--others",
+ "--exclude-standard",
+ "-z",
+ text=False,
+ )
+ untracked.check_returncode()
+ combined = dict(staged)
+ combined.update(unstaged)
+ combined.update(
+ (repo / os.fsdecode(relative), "A")
+ for relative in untracked.stdout.split(b"\0")
+ if relative and not relative.endswith(b"/")
+ )
+ return sorted(
+ (path, status)
+ for path, status in combined.items()
+ if SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts)
+ and (status == "D" or (not path.is_symlink() and path.is_file()))
+ )
+ raise SystemExit(f"Unknown diff mode: {mode}")
+
+
def candidate_source_lines(
target: Path,
diff_target: dict[str, str],
@@ -318,8 +389,6 @@ def candidate_source_lines(
case_insensitive: bool = False,
) -> dict[str, dict[str, Any]]:
"""Resolve candidate sources in the same selected view as the diff inventory."""
- from generate_rank_input import git_changed_paths
-
base, head = diff_target["baseRevision"], diff_target["headRevision"]
local = diff_target["kind"] == "working_tree"
changed = {
diff --git a/plugins/codex-security/skills/security-scan/SKILL.md b/plugins/codex-security/skills/security-scan/SKILL.md
index b0778f6b1..7c90db2ac 100644
--- a/plugins/codex-security/skills/security-scan/SKILL.md
+++ b/plugins/codex-security/skills/security-scan/SKILL.md
@@ -19,7 +19,7 @@ SDK hosts supply their own progress instructions. Use actual deduplicated review
## Workflow
-1. Resolve the authorized target, scope, scan directory, and source context. Preserve the supplied threat model, applicable inherited `SECURITY.md` guidance, and optional `CODEX_SECURITY_KNOWLEDGE_BASE`. When `CODEX_SECURITY_TARGET_PATHS_FILE` is supplied, use the SDK's exact `make-repo-scope-input` command to resolve every requested file and non-ignored directory descendant before review. Pass the resulting inventory to the core audit without widening scope; do not print, modify, or evaluate the scope file.
+1. Resolve the authorized target, scope, scan directory, and source context. Preserve the supplied threat model, applicable inherited `SECURITY.md` guidance, and optional `CODEX_SECURITY_KNOWLEDGE_BASE`. When `CODEX_SECURITY_TARGET_PATHS_FILE` is supplied, use the SDK's exact `make-repo-scope-input` command to resolve every requested file and non-ignored directory descendant before review. If the host does not supply the command, run `<plugin_dir>/scripts/launch_codex_security_mcp --helper make-repo-scope-input --repo <repo_root> --scopes-file "$CODEX_SECURITY_TARGET_PATHS_FILE" --out <scan_dir>/scoped-source-input.jsonl` or use the [PowerShell command](#windows-scoped-input) on Windows. Pass the resulting inventory to the core audit without widening scope; do not print, modify, or evaluate the scope file.
2. Read `../../references/core-scan.md` and perform the audit within the resolved scope and configured limits. Retain its semantic `scope`, `threatModel`, `findings`, and `coverage`, including source evidence, validation, attack paths, calibrated severity/confidence, and unresolved work.
3. Save semantic checkpoints throughout the audit as described below. For an SDK-owned scan with a bound semantic draft tool, save the first `complete: false` checkpoint as soon as the threat model is available, then checkpoint new findings and decisions and submit `complete: true` when review finishes. Without that tool, retain the model in an early unsealed canonical draft and update `scan-manifest.json`, `findings.json`, and `coverage.json` using `../../references/scan-contract.md` and the SDK's exact identity and directory. Omit `scan.sealedAt` and `scan.artifacts`. For requested scopes, use `scoped_path` for coverage mode and inventory strategy; otherwise use `repository` mode with `directory` strategy for non-Git targets and `repository` strategy for Git targets. On SDK file-authored paths, run the supplied scope-binding command when a target-paths file is present.
4. Verify all three canonical JSON files exist. Return control for an SDK-owned scan; the SDK owns finalization and report generation. For other host-backed scans, call `complete_codex_security_scan({ scanId, handoffClaimToken? })` once and return after it succeeds and `report.md` exists. If completion fails, surface its exact error and preserve the scan for continuation. Do not hand-author reports or reread complete canonical findings unless requested. Report complete measured token usage only; leave pending usage to the host and identify unavailable counts.
@@ -44,3 +44,15 @@ cmd.exe /d /v:off /s /c '""%scopePluginRoot%\scripts\launch_codex_security_mcp.c
```
These are temporary shell variables. Resolve paths against PowerShell's current location, including UNC shares. Missing paths stop the command. CMD expands these references once, preserving literal `%` and `!` in paths.
+
+## Windows Scoped Input
+
+Use the following PowerShell command to generate the source inventory when the host does not supply one.
+
+```powershell
+$env:scopePluginRoot = Convert-Path -LiteralPath '<plugin_dir>' -ErrorAction Stop
+$env:scopeRepository = Convert-Path -LiteralPath '<repo_root>' -ErrorAction Stop
+$env:scopeScanDir = Convert-Path -LiteralPath '<scan_dir>' -ErrorAction Stop
+$env:scopeTargetPaths = Convert-Path -LiteralPath $env:CODEX_SECURITY_TARGET_PATHS_FILE -ErrorAction Stop
+cmd.exe /d /v:off /s /c '""%scopePluginRoot%\scripts\launch_codex_security_mcp.cmd" --helper make-repo-scope-input --repo "%scopeRepository%\." --scopes-file "%scopeTargetPaths%" --out "%scopeScanDir%\scoped-source-input.jsonl""'
+```
diff --git a/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md b/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md
index d206b95a0..24707702b 100644
--- a/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md
+++ b/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md
@@ -7,11 +7,11 @@ Use this reference only for standalone or legacy diff discovery without a compac
Generate `rank_input.jsonl` from every changed text file, including deleted files. Tracked changes remain in scope regardless of ignore rules; local patches also include non-ignored untracked files.
```text
-<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode revisions --head <head> --out <discovery_dir>/rank_input.jsonl
-<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode local-patch --out <discovery_dir>/rank_input.jsonl
+<plugin_dir>/scripts/launch_codex_security_mcp --helper make-diff-rank-input --repo <repo_root> --base <base> --mode revisions --head <head> --out <discovery_dir>/rank_input.jsonl
+<plugin_dir>/scripts/launch_codex_security_mcp --helper make-diff-rank-input --repo <repo_root> --base <base> --mode local-patch --out <discovery_dir>/rank_input.jsonl
```
-Choose the first command for PR, commit, or branch diffs, and the second for a local patch. Resolve `<python_command>` from the configured interpreter as described in the shared path reference. Keep the exact baseline and target unchanged.
+Choose the first command for PR, commit, or branch diffs, and the second for a local patch. On Windows, use the [PowerShell generation commands](#windows-diff-worklist-generation). Keep the exact baseline and target unchanged.
Copy every row to the review worklist; diff scans do not rank or drop changed files:
@@ -19,7 +19,7 @@ Copy every row to the review worklist; diff scans do not rank or drop changed fi
<plugin_dir>/scripts/launch_codex_security_mcp --helper copy-deep-review-input --rank-input <discovery_dir>/rank_input.jsonl --out <discovery_dir>/deep_review_input.jsonl
```
-On Windows, use the PowerShell command below. Run file-producing helpers in managed temporary storage and import retained output through the artifact tool. In these commands, `<discovery_dir>` is the prepared helper-output directory, not an invitation to write directly into retained storage.
+On Windows, use the [PowerShell copy command](#windows-worklist-copy). Run file-producing helpers in managed temporary storage and import retained output through the artifact tool. In these commands, `<discovery_dir>` is the prepared helper-output directory, not an invitation to write directly into retained storage.
## Scoped Deep Review
@@ -52,3 +52,30 @@ cmd.exe /d /v:off /s /c '""%worklistPluginRoot%\scripts\launch_codex_security_mc
```
These are temporary shell variables. Resolve paths against PowerShell's current location, including UNC shares. The input directory and `rank_input.jsonl` must exist; the output file need not. CMD expands these references once, preserving literal `%` and `!` in paths.
+
+## Windows Diff Worklist Generation
+
+Use literal paths as in the copy command and the full commit object IDs resolved for the scan target.
+
+For PR, commit or branch diffs, run this command.
+
+```powershell
+$env:worklistPluginRoot = Convert-Path -LiteralPath '<plugin_dir>' -ErrorAction Stop
+$env:worklistRepoRoot = Convert-Path -LiteralPath '<repo_root>' -ErrorAction Stop
+$env:worklistDiscoveryDir = Convert-Path -LiteralPath '<discovery_dir>' -ErrorAction Stop
+$env:worklistBase = '<resolved_base_object_id>'
+$env:worklistHead = '<resolved_head_object_id>'
+cmd.exe /d /v:off /s /c '""%worklistPluginRoot%\scripts\launch_codex_security_mcp.cmd" --helper make-diff-rank-input --repo "%worklistRepoRoot%\." --base "%worklistBase%" --mode revisions --head "%worklistHead%" --out "%worklistDiscoveryDir%\rank_input.jsonl""'
+```
+
+For a local patch, run this command.
+
+```powershell
+$env:worklistPluginRoot = Convert-Path -LiteralPath '<plugin_dir>' -ErrorAction Stop
+$env:worklistRepoRoot = Convert-Path -LiteralPath '<repo_root>' -ErrorAction Stop
+$env:worklistDiscoveryDir = Convert-Path -LiteralPath '<discovery_dir>' -ErrorAction Stop
+$env:worklistBase = '<resolved_base_object_id>'
+cmd.exe /d /v:off /s /c '""%worklistPluginRoot%\scripts\launch_codex_security_mcp.cmd" --helper make-diff-rank-input --repo "%worklistRepoRoot%\." --base "%worklistBase%" --mode local-patch --out "%worklistDiscoveryDir%\rank_input.jsonl""'
+```
+
+The repository’s `\.` suffix preserves drive-root paths. Use the resolved object IDs because quotes in a revision expression can change CMD argument parsing.
diff --git a/plugins/codex-security/tests/test_cli_dry_runs.py b/plugins/codex-security/tests/test_cli_dry_runs.py
index f51fbdfea..4cd4230f2 100644
--- a/plugins/codex-security/tests/test_cli_dry_runs.py
+++ b/plugins/codex-security/tests/test_cli_dry_runs.py
@@ -25,8 +25,6 @@ def run_script(name: str, *args: str, isolated: bool = False) -> subprocess.Comp
(
"deep_scan_config.py",
"finalize_scan_contract.py",
- "generate_in_scope_files.py",
- "generate_rank_input.py",
"snapshot_sqlite.py",
"threat_model_projection.py",
"validate_scan_contract.py",
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
deleted file mode 100644
index d25ce685b..000000000
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ /dev/null
@@ -1,890 +0,0 @@
-from __future__ import annotations
-
-import errno
-import json
-import os
-import runpy
-import subprocess
-import sys
-from pathlib import Path, PurePosixPath
-
-import pytest
-from source_cases import SOURCE_CASES
-
-SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "generate_in_scope_files.py"
-
-
-def write_file(repository: Path, name: str, content: bytes = b"example\n") -> Path:
- path = repository / name
- path.parent.mkdir(parents=True, exist_ok=True)
- path.write_bytes(content)
- return path
-
-
-def make_repository(tmp_path: Path) -> Path:
- repository = tmp_path / "repository"
- repository.mkdir()
- subprocess.run(
- ["git", "init", "-q"],
- cwd=repository,
- capture_output=True,
- check=True,
- )
- for name in (
- "app/routes.py",
- "app/name with spaces.py",
- "app/résumé.py",
- "app/évidence.py",
- "tests/demo.py",
- "fixtures/example.py",
- ".hidden.py",
- ".hidden-directory/handler.py",
- ):
- write_file(repository, name)
- write_file(repository, "app/binary.dat", b"\x00\xff\x01")
- write_file(repository, ".gitignore", b"ignored/\n*.skip\n")
- write_file(repository, "ignored/secret.py")
- write_file(repository, "app/ignored.skip")
- return repository
-
-
-def run_inventory(
- repository: Path,
- scope: str,
- output: Path,
- *,
- env: dict[str, str] | None = None,
- arguments: list[str] | None = None,
-) -> subprocess.CompletedProcess[str]:
- return subprocess.run(
- [
- sys.executable,
- str(SCRIPT),
- "--repo",
- str(repository),
- "--scope",
- scope,
- "--out",
- str(output),
- *(arguments or []),
- ],
- capture_output=True,
- text=True,
- env=env,
- check=False,
- )
-
-
-def standard_inventory(repository: Path, scope: str) -> bytes:
- files = subprocess.run(
- [
- "rg",
- "--files",
- "--hidden",
- "--glob",
- "!**/.git",
- "--glob",
- "!**/.git/**",
- "--path-separator=/",
- "--",
- scope,
- ],
- cwd=repository,
- capture_output=True,
- check=False,
- )
- assert files.returncode in (0, 1), files.stderr.decode("utf-8", errors="replace")
- return b"".join(sorted(files.stdout.splitlines(keepends=True)))
-
-
-def git(repository: Path, *arguments: str) -> str:
- result = subprocess.run(
- ["git", "-c", "user.name=Fixture", "-c", "[email protected]", *arguments],
- cwd=repository,
- capture_output=True,
- text=True,
- check=True,
- )
- return result.stdout.strip()
-
-
[email protected]("scope", [".", "app", "./app", "app/routes.py"])
-def test_inventory_matches_the_existing_standard_command(tmp_path: Path, scope: str) -> None:
- repository = make_repository(tmp_path)
- output = tmp_path / "artifacts" / "02_discovery" / "in_scope_files.txt"
- output.parent.mkdir(parents=True)
- output.write_text("previous.py\n", encoding="utf-8")
-
- result = run_inventory(repository, scope, output)
-
- assert result.returncode == 0, result.stderr
- assert output.read_bytes() == standard_inventory(repository, scope)
- assert list(output.parent.glob(".*.tmp")) == []
- if scope == ".":
- rows = set(output.read_text(encoding="utf-8").splitlines())
- assert {
- "./.hidden.py",
- "./.hidden-directory/handler.py",
- "./tests/demo.py",
- "./fixtures/example.py",
- "./app/binary.dat",
- "./app/name with spaces.py",
- "./app/résumé.py",
- "./app/évidence.py",
- } <= rows
- assert {"./ignored/secret.py", "./app/ignored.skip"}.isdisjoint(rows)
-
-
- "name", [f"{'a' * 251}.txt", f"{'文' * 83}.txt"], ids=["ascii", "multibyte"]
-)
-def test_inventory_replaces_outputs_with_long_filenames(tmp_path: Path, name: str) -> None:
- repository = make_repository(tmp_path)
- output = tmp_path / name
- output.write_text("previous.py\n", encoding="utf-8")
-
- result = run_inventory(repository, ".", output)
-
- assert result.returncode == 0, result.stderr
- assert output.read_bytes() == standard_inventory(repository, ".")
- assert list(output.parent.glob(".*.tmp")) == []
-
-
-def test_absolute_scope_still_produces_repository_relative_paths(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, str(repository / "app"), output)
-
- assert result.returncode == 0, result.stderr
- assert output.read_bytes() == standard_inventory(repository, "app")
- assert all(not Path(line).is_absolute() for line in output.read_text().splitlines())
-
-
[email protected]("selected_path", [".", "component"])
[email protected]("linked_worktree", [False, True])
-def test_inventory_keeps_ignored_tracked_files_without_ignored_untracked_files(
- tmp_path: Path, selected_path: str, linked_worktree: bool
-) -> None:
- repository = make_repository(tmp_path)
- if linked_worktree:
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- worktree = tmp_path / "worktree"
- git(repository, "worktree", "add", "-q", str(worktree), "HEAD")
- repository = worktree
- selected = repository / selected_path
- write_file(selected, "ignored/tracked.py")
- write_file(selected, "ignored/untracked.py")
- git(selected, "add", "--force", "--", "ignored/tracked.py")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(selected, ".", output)
-
- assert result.returncode == 0, result.stderr
- paths = set(output.read_text(encoding="utf-8").splitlines())
- assert "./ignored/tracked.py" in paths
- assert "./ignored/untracked.py" not in paths
- assert "./ignored/secret.py" not in paths
- assert "./app/ignored.skip" not in paths
-
-
[email protected]("scope", [".", ".svn", "vendor/.svn/pristine/base"])
-def test_inventory_excludes_svn_metadata_even_when_tracked(tmp_path: Path, scope: str) -> None:
- repository = make_repository(tmp_path)
- write_file(repository, ".svn/wc.db")
- write_file(repository, "vendor/.svn/pristine/base")
- write_file(repository, ".gitignore", b".svn/\n")
- git(repository, "add", "--force", ".svn", "vendor/.svn")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, scope, output)
-
- assert result.returncode == 0, result.stderr
- paths = output.read_text(encoding="utf-8").splitlines()
- assert all(".svn" not in PurePosixPath(path).parts for path in paths)
- if scope == ".":
- assert "./app/routes.py" in paths
- else:
- assert paths == []
-
-
- ("scope", "absolute"),
- [
- (".", False),
- ("ignored", False),
- ("./ignored", False),
- ("ignored/tracked.py", False),
- ("./ignored/tracked.py", False),
- ("ignored", True),
- ("ignored/tracked.py", True),
- ],
-)
-def test_inventory_lists_ignored_tracked_files_once(
- tmp_path: Path, scope: str, absolute: bool
-) -> None:
- repository = make_repository(tmp_path)
- write_file(repository, "ignored/tracked.py")
- git(repository, "add", "--force", "--", "ignored/tracked.py")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, str(repository / scope) if absolute else scope, output)
-
- assert result.returncode == 0, result.stderr
- rows = output.read_bytes().splitlines()
- prefix = b"./" if scope == "." or scope.startswith("./") else b""
- assert rows.count(prefix + b"ignored/tracked.py") == 1
- assert rows == sorted(set(rows))
- assert result.stdout == f"Recorded {len(rows)} in-scope files.\n"
-
-
- ("scope", "expected"),
- [(".", "./vendor/lib/handler.py"), ("vendor", "vendor/lib/handler.py")],
-)
-def test_inventory_excludes_git_metadata_from_a_nested_repository(
- tmp_path: Path, scope: str, expected: str
-) -> None:
- repository = make_repository(tmp_path)
- write_file(repository, "vendor/lib/handler.py")
- write_file(repository, "vendor/lib/.git/HEAD", b"ref: refs/heads/main\n")
- write_file(repository, "vendor/lib/.git/config", b"[core]\n\tbare = false\n")
- write_file(repository, "vendor/lib/.git/objects/ab/cdef", b"blob")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, scope, output)
-
- assert result.returncode == 0, result.stderr
- paths = output.read_text(encoding="utf-8").splitlines()
- assert expected in paths
- assert all(".git" not in PurePosixPath(path).parts for path in paths)
-
-
- "scope",
- [
- ".git",
- "./.git",
- ".git/hooks",
- "vendor/lib/.git",
- "./vendor/lib/.git",
- "vendor/lib/.git/hooks",
- ],
-)
-def test_inventory_excludes_metadata_when_scoped_inside_a_git_directory(
- tmp_path: Path, scope: str
-) -> None:
- repository = make_repository(tmp_path)
- for prefix in (".git", "vendor/lib/.git"):
- write_file(repository, f"{prefix}/hooks/example.py")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, scope, output)
-
- assert result.returncode == 0, result.stderr
- assert output.read_bytes() == b""
-
-
[email protected]("nested", [False, True])
-def test_inventory_excludes_the_git_file_of_a_linked_worktree(tmp_path: Path, nested: bool) -> None:
- repository = make_repository(tmp_path)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- worktree = repository / "vendor" / "lib" if nested else tmp_path / "worktree"
- git(repository, "worktree", "add", "-q", str(worktree), "HEAD")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository if nested else worktree, ".", output)
-
- assert result.returncode == 0, result.stderr
- paths = output.read_text(encoding="utf-8").splitlines()
- prefix = "./vendor/lib" if nested else "."
- assert f"{prefix}/app/routes.py" in paths
- assert all(".git" not in PurePosixPath(path).parts for path in paths)
-
-
[email protected](os.name == "nt", reason="Windows does not allow CR/LF in filenames")
- ("scope_kind", "separator"),
- [
- pytest.param("ignored-tracked", "\r", id="git-ignored-tracked-cr"),
- pytest.param("directory", "\n", id="ripgrep-directory-lf"),
- pytest.param("file", "\r\n", id="explicit-file-crlf"),
- pytest.param("local-diff", "\n", id="diff-path-lf"),
- ],
-)
-def test_inventory_rejects_line_breaks_before_serializing_paths(
- tmp_path: Path, scope_kind: str, separator: str
-) -> None:
- repository = tmp_path / "repository"
- repository.mkdir()
- git(repository, "init", "-q")
- directory = "ignored" if scope_kind == "ignored-tracked" else "app"
- name = f"{directory}/concealed.py{separator}phantom.py"
- write_file(repository, name)
- scope = "."
- arguments = None
- if scope_kind == "ignored-tracked":
- write_file(repository, ".gitignore", b"ignored/\n")
- git(repository, "add", "--force", "--", name)
- elif scope_kind == "directory":
- scope = f"./{directory}"
- elif scope_kind == "file":
- scope = name
- else:
- git(repository, "commit", "--allow-empty", "-qm", "base")
- arguments = ["--diff-base", "HEAD", "--diff-mode", "local-patch"]
- output = tmp_path / "in_scope_files.txt"
- previous = b"previous.py\n"
- output.write_bytes(previous)
-
- result = run_inventory(repository, scope, output, arguments=arguments)
-
- assert result.returncode == 2
- assert "path that cannot fit in the file inventory" in result.stderr
- assert result.stdout == ""
- assert output.read_bytes() == previous
- assert list(output.parent.glob(".*.tmp")) == []
-
-
-def test_diff_inventory_includes_power_shell_files(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- monkeypatch.delenv("CODEX_SECURITY_GIT", raising=False)
- repository = tmp_path / "repository"
- repository.mkdir()
- subprocess.run(
- ["git", "init", "-q"],
- cwd=repository,
- capture_output=True,
- check=True,
- )
- git(repository, "commit", "--allow-empty", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
-
- write_file(repository, "config.json", b'{"enabled": true}\n')
- write_file(repository, "build.ps1", b'Write-Output "build"\n')
- write_file(repository, "module.psm1", b"function Get-Example { 'example' }\n")
- write_file(
- repository,
- "module.psd1",
- b"@{ RootModule = 'module.psm1'; ModuleVersion = '1.0.0' }\n",
- )
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "add diff files")
- head = git(repository, "rev-parse", "HEAD")
-
- output = tmp_path / "in_scope_files.txt"
- result = run_inventory(
- repository,
- ".",
- output,
- arguments=["--diff-base", base, "--diff-head", head],
- )
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == [
- "build.ps1",
- "config.json",
- "module.psd1",
- "module.psm1",
- ]
-
-
-def test_inventory_uses_forward_slashes_when_ripgrep_defaults_to_backslashes(
- tmp_path: Path,
-) -> None:
- repository = make_repository(tmp_path)
- output = tmp_path / "in_scope_files.txt"
- config = tmp_path / "ripgrep.conf"
- config.write_text("--path-separator=\\\n", encoding="utf-8")
-
- result = run_inventory(
- repository,
- ".",
- output,
- env={**os.environ, "RIPGREP_CONFIG_PATH": str(config)},
- )
-
- assert result.returncode == 0, result.stderr
- paths = output.read_text(encoding="utf-8").splitlines()
- assert "./app/routes.py" in paths
- assert all("\\" not in path for path in paths)
-
-
[email protected]("files", [[], ["app/handler.py"]])
-def test_inventory_without_git_metadata_replaces_the_previous_output(
- tmp_path: Path, files: list[str]
-) -> None:
- repository = tmp_path / "repository"
- repository.mkdir()
- for name in files:
- write_file(repository, name)
- output = tmp_path / "in_scope_files.txt"
- output.write_text("old.py\n", encoding="utf-8")
-
- result = run_inventory(repository, ".", output)
-
- assert result.returncode == 0, result.stderr
- assert output.read_bytes() == standard_inventory(repository, ".")
- assert f"{len(files)} in-scope files" in result.stdout
-
-
- ("scope", "message"),
- [
- ("missing", "--scope: path does not exist"),
- ("../outside", "--scope: path must remain inside --repo"),
- ],
-)
-def test_invalid_scope_preserves_the_previous_inventory(
- tmp_path: Path,
- scope: str,
- message: str,
-) -> None:
- repository = make_repository(tmp_path)
- (tmp_path / "outside").mkdir()
- output = tmp_path / "in_scope_files.txt"
- output.write_text("previous.py\n", encoding="utf-8")
-
- result = run_inventory(repository, scope, output)
-
- assert result.returncode == 2
- assert message in result.stderr
- assert output.read_text(encoding="utf-8") == "previous.py\n"
-
-
-def test_scope_cannot_follow_a_symlink_outside_the_repository(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- outside = tmp_path / "outside"
- outside.mkdir()
- write_file(outside, "external.py")
- try:
- (repository / "external-link").symlink_to(outside, target_is_directory=True)
- except OSError as error:
- pytest.skip(f"creating a symbolic link requires host support: {error}")
- output = tmp_path / "in_scope_files.txt"
- output.write_text("previous.py\n", encoding="utf-8")
-
- result = run_inventory(repository, "external-link", output)
-
- assert result.returncode == 2
- assert "--scope: path must remain inside --repo" in result.stderr
- assert output.read_text(encoding="utf-8") == "previous.py\n"
-
-
-def test_inventory_refuses_to_replace_an_output_symlink(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- original = tmp_path / "original.txt"
- original.write_text("previous.py\n", encoding="utf-8")
- output = tmp_path / "linked-inventory.txt"
- try:
- output.symlink_to(original)
- except OSError as error:
- pytest.skip(f"creating a symbolic link requires host support: {error}")
-
- result = run_inventory(repository, ".", output)
-
- assert result.returncode == 2
- assert "--out: refusing to replace a symbolic link" in result.stderr
- assert output.is_symlink()
- assert original.read_text(encoding="utf-8") == "previous.py\n"
-
-
-def test_missing_ripgrep_preserves_the_previous_inventory(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- output = tmp_path / "in_scope_files.txt"
- output.write_text("previous.py\n", encoding="utf-8")
- missing_programs = tmp_path / "missing-programs"
-
- result = run_inventory(
- repository,
- ".",
- output,
- env={**os.environ, "PATH": str(missing_programs)},
- )
-
- assert result.returncode == 2
- assert "could not run ripgrep" in result.stderr
- assert output.read_text(encoding="utf-8") == "previous.py\n"
-
-
[email protected](os.name == "nt", reason="Windows cannot launch executable script shims")
-def test_ripgrep_failure_preserves_the_previous_inventory(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- tools = tmp_path / "tools"
- tools.mkdir()
- ripgrep = tools / "rg"
- ripgrep.write_text(
- "#!/bin/sh\nprintf '%s\\n' 'simulated ripgrep failure' >&2\nexit 2\n",
- encoding="utf-8",
- )
- ripgrep.chmod(0o755)
- output = tmp_path / "in_scope_files.txt"
- output.write_text("previous.py\n", encoding="utf-8")
-
- result = run_inventory(repository, ".", output, env={**os.environ, "PATH": str(tools)})
-
- assert result.returncode == 2
- assert "ripgrep exited with status 2: simulated ripgrep failure" in result.stderr
- assert output.read_text(encoding="utf-8") == "previous.py\n"
-
-
[email protected](os.name == "nt", reason="Windows cannot launch executable script shims")
-def test_large_inventory_is_not_limited_by_a_subprocess_output_buffer(tmp_path: Path) -> None:
- repository = tmp_path / "repository"
- repository.mkdir()
- tools = tmp_path / "tools"
- tools.mkdir()
- ripgrep = tools / "rg"
- ripgrep.write_text(
- f"#!{sys.executable}\n"
- "import sys\n"
- "for index in range(12000, 0, -1):\n"
- " sys.stdout.write(f\"./{'x' * 100}-{index:05d}.py\\0\")\n",
- encoding="utf-8",
- )
- ripgrep.chmod(0o755)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, env={**os.environ, "PATH": str(tools)})
-
- assert result.returncode == 0, result.stderr
- assert output.stat().st_size > 1024 * 1024
- rows = output.read_bytes().splitlines()
- assert len(rows) == 12_000
- assert rows == sorted(rows)
-
-
[email protected]("mode", ["revisions", "staged", "unstaged"])
-def test_diff_inventory_keeps_changed_and_deleted_source_files(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- write_file(repository, "app/entrypoint", b"exec service\n")
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- gitlink = git(repository, "rev-parse", "HEAD")
- git(repository, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},removed-module")
- git(repository, "commit", "--amend", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
-
- write_file(repository, "app/routes.py", b"changed = True\n")
- write_file(repository, "app/new handler.py", b"handler = True\n")
- write_file(repository, "app/binary.py", b"\x00\xff\x01")
- write_file(repository, "tests/demo.py", b"changed = True\n")
- write_file(repository, ".github/workflows/ci.yml", b"name: CI\n")
- write_file(
- repository,
- ".gitmodules",
- b'[submodule "added-module"]\npath = added-module\nurl = ../module.git\n',
- )
- (repository / "app/évidence.py").unlink()
- (repository / "app/entrypoint").unlink()
- (repository / "app/binary.dat").unlink()
- arguments = ["--diff-base", base, "--diff-mode", "local-patch"]
- if mode in {"revisions", "staged"}:
- git(repository, "add", ".")
- git(repository, "update-index", "--force-remove", "removed-module")
- git(repository, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},added-module")
- if mode == "revisions":
- git(repository, "commit", "-qm", "change")
- arguments = ["--diff-base", base, "--diff-head", git(repository, "rev-parse", "HEAD")]
- git(repository, "checkout", "-q", base)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(
- repository,
- ".",
- output,
- arguments=arguments,
- )
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == [
- ".github/workflows/ci.yml",
- ".gitmodules",
- "app/entrypoint",
- "app/new handler.py",
- "app/routes.py",
- "app/évidence.py",
- "tests/demo.py",
- ]
-
-
[email protected]("mode", ["revisions", "local-patch"])
-def test_diff_inventory_only_classifies_source_bytes(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- write_file(repository, "app/text.unlisted", b"x" * (64 * 1024 + 1) + b"text")
- write_file(repository, "app/binary.py", b"x" * (64 * 1024 + 1) + b"\0")
- write_file(repository, "app/binary.unlisted", b"x" * (64 * 1024 + 1) + b"\0")
- arguments = ["--diff-base", base, "--diff-mode", mode]
- if mode == "revisions":
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "change")
- arguments.extend(["--diff-head", "HEAD"])
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == ["app/text.unlisted"]
-
-
[email protected]("mode", ["revisions", "staged", "unstaged"])
-def test_diff_inventory_includes_source_cases(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- for case in SOURCE_CASES:
- write_file(repository, case.path, (case.before + "\n").encode("utf-8"))
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- for case in SOURCE_CASES:
- write_file(repository, case.path, (case.after + "\n").encode("utf-8"))
- arguments = ["--diff-base", base, "--diff-mode", "local-patch"]
- if mode in {"revisions", "staged"}:
- git(repository, "add", ".")
- if mode == "revisions":
- git(repository, "commit", "-qm", "change")
- arguments = ["--diff-base", base, "--diff-head", git(repository, "rev-parse", "HEAD")]
- git(repository, "checkout", "-q", base)
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == sorted(
- case.path for case in SOURCE_CASES
- )
-
-
-def test_diff_inventory_combines_staged_and_unstaged_changes(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
-
- write_file(repository, "app/routes.py", b"unstaged = True\n")
- write_file(repository, "app/staged.py", b"staged = True\n")
- git(repository, "add", "app/staged.py")
- write_file(repository, "app/untracked.py", b"untracked = True\n")
- write_file(repository, ".github/workflows/ci.yaml", b"name: CI\n")
- write_file(repository, "app/untracked-binary.py", b"\x00\xff\x01")
- index_only = write_file(repository, "app/index-only.py", b"index_only = True\n")
- git(repository, "add", "app/index-only.py")
- index_only.unlink()
- nested = repository / "nested"
- nested.mkdir()
- git(nested, "init", "-q")
- write_file(nested, "source.py", b"nested = True\n")
- git(nested, "add", ".")
- git(nested, "commit", "-qm", "nested repository")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(
- repository,
- ".",
- output,
- arguments=["--diff-base", "HEAD", "--diff-mode", "local-patch"],
- )
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == [
- ".github/workflows/ci.yaml",
- "app/routes.py",
- "app/staged.py",
- "app/untracked.py",
- ]
-
-
-def test_revision_inventory_checks_the_selected_revision_not_the_worktree(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
-
- write_file(repository, "app/routes.py", b"selected = True\n")
- write_file(repository, "app/name with spaces.py", b"\x00\xff\x01")
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "selected revision")
- selected = git(repository, "rev-parse", "HEAD")
-
- write_file(repository, "app/routes.py", b"\x00\xff\x01")
- write_file(repository, "app/name with spaces.py", b"current = True\n")
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "current revision")
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(
- repository,
- ".",
- output,
- arguments=["--diff-base", base, "--diff-head", selected],
- )
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == ["app/routes.py"]
-
-
[email protected]("mode", ["revisions", "local-patch"])
-def test_diff_inventory_includes_bom_marked_utf16_text(tmp_path: Path, mode: str) -> None:
- repository = make_repository(tmp_path)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- source = "Write-Output 'café'\n"
- write_file(repository, "app/utf16-le.ps1", b"\xff\xfe" + source.encode("utf-16-le"))
- write_file(repository, "app/utf16-be.ps1", b"\xfe\xff" + source.encode("utf-16-be"))
- write_file(repository, "app/utf8.ps1", source.encode("utf-8"))
- write_file(repository, "app/binary.ps1", b"text\0binary")
- write_file(
- repository,
- "app/decoded-nul.ps1",
- b"\xff\xfe" + "text\0binary".encode("utf-16-le"),
- )
- write_file(repository, "tests/encoded.ps1", b"\xff\xfe" + source.encode("utf-16-le"))
-
- arguments = ["--diff-base", base, "--diff-mode", mode]
- if mode == "revisions":
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "add encoded source")
- arguments.extend(["--diff-head", git(repository, "rev-parse", "HEAD")])
- output = tmp_path / "in_scope_files.txt"
-
- result = run_inventory(repository, ".", output, arguments=arguments)
-
- assert result.returncode == 0, result.stderr
- assert output.read_text(encoding="utf-8").splitlines() == [
- "app/utf16-be.ps1",
- "app/utf16-le.ps1",
- "app/utf8.ps1",
- "tests/encoded.ps1",
- ]
-
-
-def test_invalid_diff_revision_preserves_previous_inventory(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- output = tmp_path / "in_scope_files.txt"
- output.write_text("previous.py\n", encoding="utf-8")
-
- result = run_inventory(repository, ".", output, arguments=["--diff-base", "missing"])
-
- assert result.returncode == 2
- assert "could not resolve the selected Git changes" in result.stderr
- assert output.read_text(encoding="utf-8") == "previous.py\n"
-
-
-def test_diff_inventory_rejects_a_narrower_scope(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- output = tmp_path / "in_scope_files.txt"
- output.write_text("previous.py\n", encoding="utf-8")
-
- result = run_inventory(repository, "app", output, arguments=["--diff-base", "HEAD"])
-
- assert result.returncode == 2
- assert "diff scans must use the repository root" in result.stderr
- assert output.read_text(encoding="utf-8") == "previous.py\n"
-
-
- ("mode", "undo_staged_change"),
- [("revisions", False), ("local-patch", False), ("local-patch", True)],
-)
-def test_diff_inventory_includes_type_changed_files(
- tmp_path: Path, mode: str, undo_staged_change: bool
-) -> None:
- repository = make_repository(tmp_path)
- source = repository / "app" / "changed.py"
- try:
- source.symlink_to("routes.py")
- except OSError:
- pytest.skip("symlinks are unavailable")
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- source.unlink()
- source.write_text("changed = True\n")
- if undo_staged_change:
- git(repository, "add", ".")
- source.unlink()
- source.symlink_to("routes.py")
- arguments = ["--diff-base", "HEAD", "--diff-mode", mode]
- if mode == "revisions":
- base = git(repository, "rev-parse", "HEAD")
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "replace symlink")
- arguments = ["--diff-base", base, "--diff-head", "HEAD", "--diff-mode", mode]
- output = tmp_path / "in_scope_files.txt"
- result = run_inventory(repository, ".", output, arguments=arguments)
- assert result.returncode == 0, result.stderr
- assert (b"app/changed.py\n" in output.read_bytes()) is not undo_staged_change
- rank_output = tmp_path / "rank.jsonl"
- rank = subprocess.run(
- [
- sys.executable,
- str(SCRIPT.with_name("generate_rank_input.py")),
- "make-diff-rank-input",
- "--repo",
- str(repository),
- "--base",
- arguments[1],
- "--head",
- "HEAD",
- "--mode",
- mode,
- "--out",
- str(rank_output),
- ],
- capture_output=True,
- text=True,
- )
- assert rank.returncode == 0, rank.stderr
- assert (
- any(
- json.loads(line)["path"] == "app/changed.py"
- for line in rank_output.read_text().splitlines()
- )
- is not undo_staged_change
- )
-
-
[email protected](os.name == "nt", reason="Windows paths cannot retain arbitrary non-UTF-8 bytes")
-def test_diff_inventory_preserves_non_utf8_path_bytes(tmp_path: Path) -> None:
- repository = make_repository(tmp_path)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- base = git(repository, "rev-parse", "HEAD")
- write_file(repository, os.fsdecode(b"app/caf\xe9.py"), b"changed = True\n")
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "Add byte path")
- output = tmp_path / "in_scope_files.txt"
- result = run_inventory(
- repository, ".", output, arguments=["--diff-base", base, "--diff-head", "HEAD"]
- )
- assert result.returncode == 0, result.stderr
- assert output.read_bytes() == b"app/caf\xe9.py\n"
-
-
-def test_local_diff_inventory_skips_unreadable_files(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- inventory = runpy.run_path(str(SCRIPT))
- repository = make_repository(tmp_path)
- git(repository, "add", ".")
- git(repository, "commit", "-qm", "base")
- unreadable = write_file(repository, "app/unreadable.py")
- write_file(repository, "app/readable.py")
- output = tmp_path / "in_scope_files.txt"
- original_open = Path.open
-
- def open_file(path, *args, **kwargs):
- if path == unreadable:
- raise PermissionError(errno.EACCES, "Synthetic unreadable file", str(path))
- return original_open(path, *args, **kwargs)
-
- monkeypatch.setattr(Path, "open", open_file)
- count = inventory["generate_diff_in_scope_files"](
- repository, "HEAD", "HEAD", "local-patch", output
- )
- assert count == 1
- assert output.read_text(encoding="utf-8") == "app/readable.py\n"
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
deleted file mode 100644
index e984dd08e..000000000
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ /dev/null
@@ -1,805 +0,0 @@
-from __future__ import annotations
-
-import json
-import runpy
-import subprocess
-import sys
-from pathlib import Path
-from unittest.mock import patch
-
-import pytest
-from source_cases import SOURCE_CASES
-
-PLUGIN_ROOT = Path(__file__).resolve().parents[1]
-SCRIPT = PLUGIN_ROOT / "scripts" / "generate_rank_input.py"
-
-
-def run_repo_cli(
- command: str, repo: str | Path, output: Path, *args: str, check: bool = True
-) -> subprocess.CompletedProcess[str]:
- return subprocess.run(
- [sys.executable, str(SCRIPT), command, "--repo", str(repo), *args, "--out", str(output)],
- check=check,
- capture_output=True,
- text=True,
- )
-
-
-def test_cli_loads_preview_helper_with_safe_path() -> None:
- result = subprocess.run(
- [sys.executable, "-I", str(SCRIPT), "--help"],
- check=True,
- capture_output=True,
- text=True,
- )
-
- assert "Codex Security scan worklist helper" in result.stdout
-
-
-def read_jsonl(path: Path) -> list[dict[str, object]]:
- return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
-
-
-def git(repo: Path, *args: str, input: str | None = None) -> str:
- result = subprocess.run(
- ["git", "-C", str(repo), *args],
- check=True,
- capture_output=True,
- input=input,
- text=True,
- )
- return result.stdout.strip()
-
-
-def initialize_repo(repo: Path) -> None:
- git(repo, "init", "-q")
- git(repo, "config", "user.email", "[email protected]")
- git(repo, "config", "user.name", "Codex Security Tests")
-
-
-def test_rank_input_keeps_large_generated_python(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- initialize_repo(repo)
- git(repo, "commit", "--allow-empty", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- source = "value = " + " + ".join(["x"] * 150000) + "\n"
- (repo / "src" / "generated.py").write_text(source, encoding="utf-8")
- (repo / "src" / "normal.py").write_text("value = 1\n", encoding="utf-8")
- output = tmp_path / "rank_input.jsonl"
-
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "change")
- command = "make-diff-rank-input"
- arguments = ["--base", base, "--head", "HEAD"]
- run_repo_cli(command, repo, output, *arguments, "--preview-bytes", "128")
-
- rows = read_jsonl(output)
- assert [row["path"] for row in rows] == ["src/generated.py", "src/normal.py"]
- preview = rows[0]["preview"]
- assert preview
- assert source.startswith(preview)
- assert len(preview.encode("utf-8")) <= 128
- assert rows[1]["preview"] == "value = 1"
-
-
-def test_rank_input_streams_classification_and_bounds_previews(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- source = repo / "src" / "large.py"
- source.write_bytes(
- b"def visible():\n pass\n"
- + b"# source comment\n" * (128 * 1024)
- + b"def outside():\n pass\n"
- )
- output = tmp_path / "rank_input.jsonl"
- initialize_repo(repo)
- git(repo, "commit", "--allow-empty", "-qm", "initial")
- arguments = [
- "make-diff-rank-input",
- "--repo",
- str(repo),
- "--out",
- str(output),
- "--mode",
- "local-patch",
- "--base",
- "HEAD",
- ]
-
- original_open = Path.open
- reads: list[int] = []
-
- def tracked_open(path: Path, *args, **kwargs):
- reader = original_open(path, *args, **kwargs)
- if path == source:
- read = reader.read
- index = len(reads)
- reads.append(0)
-
- def bounded_read(size: int = -1) -> bytes:
- assert 0 <= size <= 64 * 1024
- data = read(size)
- reads[index] += len(data)
- return data
-
- reader.read = bounded_read
- return reader
-
- monkeypatch.setattr(sys, "argv", [str(SCRIPT), *arguments])
- with patch.object(Path, "open", tracked_open):
- runpy.run_path(str(SCRIPT), run_name="__main__")
-
- assert set(reads) == {64 * 1024, source.stat().st_size}
- rows = read_jsonl(output)
- assert len(rows) == 1
- assert rows[0]["path"] == "src/large.py"
- preview = rows[0]["preview"]
- assert preview.startswith("def visible():\n pass\n# source comment")
- assert len(preview.encode("utf-8")) <= 1024
- assert "outside" not in preview
-
-
[email protected]("mode", ["revisions", "staged", "unstaged"])
-def test_rank_input_includes_source_cases(tmp_path: Path, mode: str) -> None:
- repo = tmp_path / "repo"
- repo.mkdir()
- initialize_repo(repo)
- for case in SOURCE_CASES:
- source = repo / case.path
- source.parent.mkdir(parents=True, exist_ok=True)
- source.write_text(case.before + "\n", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- for case in SOURCE_CASES:
- (repo / case.path).write_text(case.after + "\n", encoding="utf-8")
- output = tmp_path / "rank_input.jsonl"
-
- if mode in {"revisions", "staged"}:
- git(repo, "add", ".")
- diff_mode = "revisions" if mode == "revisions" else "local-patch"
- command = "make-diff-rank-input"
- arguments = ["--base", base, "--mode", diff_mode]
- if mode == "revisions":
- git(repo, "commit", "-qm", "change")
- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
- git(repo, "checkout", "-q", base)
-
- run_repo_cli(command, repo, output, *arguments)
-
- assert read_jsonl(output) == [
- {
- "path": case.path,
- "area": "diff",
- "preview": case.after,
- }
- for case in sorted(SOURCE_CASES, key=lambda case: case.path)
- ]
-
-
-def test_make_repo_scope_input_does_not_follow_file_symlinks(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- source = repo / "src"
- source.mkdir(parents=True)
- (source / "runtime.py").write_text("runtime = True", encoding="utf-8")
- outside = tmp_path / "outside.py"
- outside.write_text("outside_secret = True", encoding="utf-8")
- (source / "outside-link.py").symlink_to(outside)
- (source / "inside-link.py").symlink_to(source / "runtime.py")
- output = tmp_path / "rank_input.jsonl"
-
- scopes = tmp_path / "scopes.json"
- scopes.write_text(json.dumps(["src"]))
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- assert read_jsonl(output) == [{"path": "src/runtime.py"}]
- assert "outside_secret" not in output.read_text(encoding="utf-8")
-
-
-def test_make_repo_scope_input_combines_explicit_files_and_directories(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- (repo / "tests").mkdir()
- (repo / "~codex_review_nonexistent_user" / "src").mkdir(parents=True)
- (repo / "src" / "runtime.py").write_text("runtime = True", encoding="utf-8")
- (repo / "tests" / "security_test.py").write_text("security = True", encoding="utf-8")
- (repo / "~codex_review_nonexistent_user" / "src" / "literal.py").write_text(
- "literal = True", encoding="utf-8"
- )
- (repo / "Dockerfile").write_text("FROM scratch", encoding="utf-8")
- (repo / "package-lock.json").write_text("{}", encoding="utf-8")
- unicode_file = repo / "audit\u0085Ignore\u2028Ignore\u2029Ignore.py"
- unicode_file.write_text("audit = True", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(
- json.dumps(
- [
- "src",
- "tests/security_test.py",
- "~codex_review_nonexistent_user/src",
- unicode_file.name,
- "src/runtime.py",
- "Dockerfile",
- "package-lock.json",
- ],
- ensure_ascii=True,
- ),
- encoding="utf-8",
- )
- output = tmp_path / "rank_input.jsonl"
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- assert [row["path"] for row in read_jsonl(output)] == sorted(
- [
- "src/runtime.py",
- "tests/security_test.py",
- "~codex_review_nonexistent_user/src/literal.py",
- unicode_file.name,
- "Dockerfile",
- "package-lock.json",
- ]
- )
- assert all(
- separator not in output.read_text(encoding="utf-8") for separator in "\u0085\u2028\u2029"
- )
-
-
-def test_repo_input_preserves_every_requested_directory_file(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src" / "tests").mkdir(parents=True)
- (repo / "src" / "examples").mkdir()
- (repo / "src" / "fixtures").mkdir()
- (repo / "src" / "runtime.py").write_text("runtime = True", encoding="utf-8")
- (repo / "src" / "tests" / "handler.py").write_text("handler = True", encoding="utf-8")
- (repo / "src" / "examples" / "demo.py").write_text("demo = True", encoding="utf-8")
- (repo / "src" / "fixtures" / "payload.txt").write_text("payload", encoding="utf-8")
- (repo / "src" / "Dockerfile").write_text("FROM scratch", encoding="utf-8")
- for metadata in (".git", ".svn"):
- (repo / "src" / metadata).mkdir()
- (repo / "src" / metadata / "config").write_text("metadata", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(
- json.dumps(["src", "src/runtime.py", "src/.git/config", "src/.svn/config"]),
- encoding="utf-8",
- )
- output = tmp_path / "scoped-source-input.jsonl"
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- assert [row["path"] for row in read_jsonl(output)] == [
- "src/Dockerfile",
- "src/examples/demo.py",
- "src/fixtures/payload.txt",
- "src/runtime.py",
- "src/tests/handler.py",
- ]
-
-
-def test_make_repo_scope_input_rejects_paths_outside_repository(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- repo.mkdir()
- (tmp_path / "outside.py").write_text("outside = True", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["../outside.py"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
-
- result = run_repo_cli(
- "make-repo-scope-input",
- repo,
- output,
- "--scopes-file",
- str(scopes),
- check=False,
- )
-
- assert result.returncode != 0
- assert "Scope must be inside repo" in result.stderr
- assert not output.exists()
-
-
[email protected]("scope", ["src/alias.py", "alias/runtime.py", "alias/../src/runtime.py"])
-def test_make_repo_scope_input_rejects_explicit_symlink_scopes(tmp_path: Path, scope: str) -> None:
- repo = tmp_path / "repo"
- source = repo / "src"
- source.mkdir(parents=True)
- runtime = source / "runtime.py"
- runtime.write_text("runtime = True", encoding="utf-8")
- (source / "alias.py").symlink_to(runtime)
- (repo / "alias").symlink_to(source, target_is_directory=True)
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps([scope]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
-
- result = run_repo_cli(
- "make-repo-scope-input",
- repo,
- output,
- "--scopes-file",
- str(scopes),
- check=False,
- )
-
- assert result.returncode != 0
- assert "must not contain symbolic links" in result.stderr
- assert not output.exists()
-
-
[email protected]("scope", ["src", "."])
[email protected]("replacement", ["link", "loop", "file"])
-def test_repo_input_handles_replaced_tracked_directories(
- tmp_path: Path, scope: str, replacement: str
-) -> None:
- repo = tmp_path / "repo"
- nested = repo / "src" / "nested"
- nested.mkdir(parents=True)
- initialize_repo(repo)
- (repo / ".gitignore").write_text("private/\n", encoding="utf-8")
- (repo / "src" / "handler.py").write_text("handler = True\n", encoding="utf-8")
- (nested / "service.py").write_text("service = True\n", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- private = repo / "private"
- private.mkdir()
- (private / "service.py").write_text("outside_requested_scope = True\n", encoding="utf-8")
- (nested / "service.py").unlink()
- nested.rmdir()
- if replacement == "link":
- nested.symlink_to(private, target_is_directory=True)
- elif replacement == "loop":
- nested.symlink_to(nested.name, target_is_directory=True)
- else:
- nested.write_text("replacement = True\n", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps([scope]), encoding="utf-8")
- output = tmp_path / "input.jsonl"
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- expected = ["src/handler.py"]
- if replacement == "file":
- expected.append("src/nested")
- if scope == ".":
- expected.insert(0, ".gitignore")
- assert [row["path"] for row in read_jsonl(output)] == expected
-
-
-def test_repo_input_preserves_tracked_ignored_and_binary_files(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- source = repo / "src"
- vendor = source / "vendor"
- vendor.mkdir(parents=True)
- initialize_repo(repo)
- (repo / ".gitignore").write_text("vendor/\n", encoding="utf-8")
- (source / "logo.png").write_bytes(b"\x89PNG\x00")
- (vendor / "dependency.py").write_text("dependency = True", encoding="utf-8")
- git(repo, "add", "--force", "src/logo.png", "src/vendor/dependency.py")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["src"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- expected = ["src/logo.png", "src/vendor/dependency.py"]
- assert [row["path"] for row in read_jsonl(output)] == expected
-
-
-def test_repo_input_respects_ignored_directory_descendants(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src" / "node_modules").mkdir(parents=True)
- (repo / "src" / "nested").mkdir()
- (repo / ".gitignore").write_text("node_modules/\n.env\n", encoding="utf-8")
- (repo / "src" / "nested" / ".gitignore").write_text("*.generated\n", encoding="utf-8")
- (repo / "src" / "handler.py").write_text("handler = True", encoding="utf-8")
- (repo / "src" / ".env").write_text("SECRET=private", encoding="utf-8")
- (repo / "src" / "node_modules" / "dependency.js").write_text("dependency", encoding="utf-8")
- (repo / "src" / "nested" / "source.py").write_text("source = True", encoding="utf-8")
- (repo / "src" / "nested" / "output.generated").write_text("generated", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["src"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- assert {row["path"] for row in read_jsonl(output)} == {
- "src/handler.py",
- "src/nested/.gitignore",
- "src/nested/source.py",
- }
-
-
-def test_repo_input_keeps_explicitly_requested_ignored_file(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- (repo / ".gitignore").write_text(".env\n", encoding="utf-8")
- (repo / "src" / ".env").write_text("SECRET=private", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["src/.env"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- assert [row["path"] for row in read_jsonl(output)] == ["src/.env"]
-
-
-def test_repo_input_uses_git_ignore_rules(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- initialize_repo(repo)
- (repo / ".gitignore").write_text(".env\n", encoding="utf-8")
- (repo / "src" / "handler.py").write_text("handler = True", encoding="utf-8")
- (repo / "src" / ".env").write_text("SECRET=private", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["src"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- assert [row["path"] for row in read_jsonl(output)] == ["src/handler.py"]
-
-
-def test_repo_input_falls_back_without_git_or_ripgrep(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- (repo / "src" / "handler.py").write_text("handler = True", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["src"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
- monkeypatch.setenv("PATH", str(tmp_path / "missing-tools"))
-
- run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))
-
- assert [row["path"] for row in read_jsonl(output)] == ["src/handler.py"]
-
-
-def test_repo_input_fails_closed_when_ignore_rules_cannot_be_applied(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- (repo / ".gitignore").write_text(".env\n", encoding="utf-8")
- (repo / "src" / ".env").write_text("SECRET=private", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["src"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
- monkeypatch.setenv("PATH", str(tmp_path / "missing-tools"))
-
- result = run_repo_cli(
- "make-repo-scope-input", repo, output, "--scopes-file", str(scopes), check=False
- )
-
- assert result.returncode != 0
- assert "without Git or ripgrep" in result.stderr
- assert not output.exists()
-
-
-def test_repo_input_fails_closed_for_git_private_excludes_without_tools(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- repo = tmp_path / "repo"
- (repo / ".git" / "info").mkdir(parents=True)
- (repo / "src").mkdir()
- (repo / ".git" / "info" / "exclude").write_text(".env\n", encoding="utf-8")
- (repo / "src" / ".env").write_text("SECRET=private", encoding="utf-8")
- scopes = tmp_path / "target-paths.json"
- scopes.write_text(json.dumps(["src"]), encoding="utf-8")
- output = tmp_path / "scoped-source-input.jsonl"
- monkeypatch.setenv("PATH", str(tmp_path / "missing-tools"))
-
- result = run_repo_cli(
- "make-repo-scope-input", repo, output, "--scopes-file", str(scopes), check=False
- )
-
- assert result.returncode != 0
- assert "without Git or ripgrep" in result.stderr
- assert not output.exists()
-
-
[email protected]("mode", ["revisions", "staged", "unstaged"])
-def test_make_diff_rank_input_keeps_changed_and_deleted_text(tmp_path: Path, mode: str) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- initialize_repo(repo)
- (repo / "src" / "alpha.py").write_text("alpha = 1", encoding="utf-8")
- deleted_guard = repo / "src" / "deleted_guard.py"
- deleted_guard.write_text("guard = True", encoding="utf-8")
- deleted_entrypoint = repo / "src" / "entrypoint"
- deleted_entrypoint.write_text("exec service", encoding="utf-8")
- deleted_binary = repo / "src" / "deleted.bin"
- deleted_binary.write_bytes(b"value\x00binary")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- gitlink = git(repo, "rev-parse", "HEAD")
- git(repo, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},removed-module")
- git(repo, "commit", "--amend", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
-
- (repo / "src" / "alpha.py").write_text("alpha = 2", encoding="utf-8")
- (repo / "src" / "beta.py").write_text("beta = 1", encoding="utf-8")
- (repo / "src" / "binary.unlisted").write_bytes(b"x" * (64 * 1024 + 1) + b"\0binary")
- (repo / ".github" / "workflows").mkdir(parents=True)
- (repo / ".github" / "workflows" / "ci.yml").write_text("name: CI", encoding="utf-8")
- (repo / ".gitmodules").write_text(
- '[submodule "added-module"]\npath = added-module\nurl = ../module.git\n',
- encoding="utf-8",
- )
- deleted_guard.unlink()
- deleted_entrypoint.unlink()
- deleted_binary.unlink()
- (repo / "README.md").write_text("Deployment instructions", encoding="utf-8")
- arguments = ["--mode", "local-patch"]
- if mode in {"revisions", "staged"}:
- git(repo, "add", ".")
- git(repo, "update-index", "--force-remove", "removed-module")
- git(repo, "update-index", "--add", "--cacheinfo", f"160000,{gitlink},added-module")
- if mode == "revisions":
- link_blob = git(repo, "hash-object", "-w", "--stdin", input="../outside")
- git(repo, "update-index", "--add", "--cacheinfo", f"120000,{link_blob},tests/link.py")
- git(repo, "commit", "-qm", "change")
- arguments = ["--head", git(repo, "rev-parse", "HEAD")]
- git(repo, "checkout", "-q", base)
- output = tmp_path / "diff.jsonl"
-
- run_repo_cli("make-diff-rank-input", repo, output, "--base", base, *arguments)
-
- rows = read_jsonl(output)
- assert [row["path"] for row in rows] == [
- ".github/workflows/ci.yml",
- ".gitmodules",
- "README.md",
- "src/alpha.py",
- "src/beta.py",
- "src/deleted_guard.py",
- "src/entrypoint",
- ]
- assert [row["preview"] for row in rows[-2:]] == ["", ""]
-
-
-def test_make_diff_rank_input_uses_empty_tree_for_root_commit(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- initialize_repo(repo)
- (repo / "src" / "root.py").write_text("root = True", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "root")
- head = git(repo, "rev-parse", "HEAD")
- empty_tree = git(repo, "hash-object", "-t", "tree", "--stdin", input="")
- output = tmp_path / "root.jsonl"
-
- run_repo_cli("make-diff-rank-input", repo, output, "--base", empty_tree, "--head", head)
-
- assert [row["path"] for row in read_jsonl(output)] == ["src/root.py"]
-
-
-def test_make_diff_rank_input_supports_shallow_tips_without_merge_base(
- tmp_path: Path,
-) -> None:
- source = tmp_path / "source"
- (source / "src").mkdir(parents=True)
- initialize_repo(source)
- (source / "src" / "base.py").write_text("base = True", encoding="utf-8")
- git(source, "add", ".")
- git(source, "commit", "-qm", "base")
- git(source, "branch", "-M", "main")
- git(source, "checkout", "-qb", "feature")
- (source / "src" / "feature.py").write_text("feature = True", encoding="utf-8")
- git(source, "add", ".")
- git(source, "commit", "-qm", "feature")
- git(source, "checkout", "-q", "main")
- (source / "src" / "upstream.py").write_text("upstream = True", encoding="utf-8")
- git(source, "add", ".")
- git(source, "commit", "-qm", "upstream")
-
- shallow = tmp_path / "shallow"
- subprocess.run(
- [
- "git",
- "clone",
- "--no-local",
- "--depth=1",
- "--branch",
- "feature",
- str(source),
- str(shallow),
- ],
- check=True,
- capture_output=True,
- text=True,
- )
- git(shallow, "fetch", "--depth=1", "origin", "main:refs/remotes/origin/main")
- merge_base = subprocess.run(
- ["git", "-C", str(shallow), "merge-base", "origin/main", "HEAD"],
- check=False,
- capture_output=True,
- text=True,
- )
- assert merge_base.returncode == 1
- output = tmp_path / "shallow.jsonl"
-
- run_repo_cli("make-diff-rank-input", shallow, output, "--base", "origin/main", "--head", "HEAD")
-
- assert [row["path"] for row in read_jsonl(output)] == [
- "src/feature.py",
- "src/upstream.py",
- ]
-
-
-def test_make_diff_rank_input_combines_staged_and_unstaged_patch(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "src").mkdir(parents=True)
- initialize_repo(repo)
- (repo / "src" / "alpha.py").write_text("alpha = 1", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
-
- (repo / "src" / "alpha.py").write_text("alpha = 2", encoding="utf-8")
- (repo / "src" / "beta.py").write_text("beta = 1", encoding="utf-8")
- (repo / ".github" / "workflows").mkdir(parents=True)
- (repo / ".github" / "workflows" / "ci.yaml").write_text("name: CI", encoding="utf-8")
- git(repo, "add", "src/beta.py")
- index_only = repo / "src" / "index-only.py"
- index_only.write_text("index_only = True", encoding="utf-8")
- git(repo, "add", "src/index-only.py")
- index_only.unlink() # The selected working tree no longer contains this staged addition.
- nested = repo / "nested"
- nested.mkdir()
- initialize_repo(nested)
- (nested / "source.py").write_text("nested = True", encoding="utf-8")
- git(nested, "add", ".")
- git(nested, "commit", "-qm", "nested repository")
- output = tmp_path / "patch.jsonl"
-
- run_repo_cli("make-diff-rank-input", repo, output, "--base", "HEAD", "--mode", "local-patch")
-
- assert [row["path"] for row in read_jsonl(output)] == [
- ".github/workflows/ci.yaml",
- "src/alpha.py",
- "src/beta.py",
- ]
-
-
-def test_make_diff_rank_input_accepts_base_that_matches_a_path(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- (repo / "release").mkdir(parents=True)
- initialize_repo(repo)
- (repo / "release" / "build.py").write_text("build = 1", encoding="utf-8")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- git(repo, "branch", "release")
- (repo / "release" / "build.py").write_text("build = 2", encoding="utf-8")
- output = tmp_path / "patch.jsonl"
-
- run_repo_cli("make-diff-rank-input", repo, output, "--base", "release", "--mode", "local-patch")
-
- assert [row["path"] for row in read_jsonl(output)] == ["release/build.py"]
-
-
[email protected]("mode", ["revisions", "local-patch"])
-def test_make_rank_input_decodes_bom_marked_utf16_source(tmp_path: Path, mode: str) -> None:
- repo = tmp_path / "repo"
- source_dir = repo / "src"
- source_dir.mkdir(parents=True)
- initialize_repo(repo)
- git(repo, "commit", "--allow-empty", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- source = "Write-Output 'café 😀'\n"
- (source_dir / "utf16-le.ps1").write_bytes(b"\xff\xfe" + source.encode("utf-16-le"))
- (source_dir / "utf16-be.ps1").write_bytes(b"\xfe\xff" + source.encode("utf-16-be"))
- (source_dir / "utf8.ps1").write_bytes(source.encode("utf-8"))
- (source_dir / "binary.ps1").write_bytes(b"text\0binary")
- (source_dir / "decoded-nul.ps1").write_bytes(b"\xff\xfe" + "text\0binary".encode("utf-16-le"))
- output = tmp_path / "rank_input.jsonl"
- expected = {
- "src/utf16-be.ps1": source.strip(),
- "src/utf16-le.ps1": source.strip(),
- "src/utf8.ps1": source.strip(),
- }
-
- command = "make-diff-rank-input"
- arguments = ["--base", base, "--mode", mode]
- if mode == "revisions":
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "add encoded source")
- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])
- git(repo, "checkout", "-q", base)
-
- run_repo_cli(command, repo, output, *arguments)
-
- assert {row["path"]: row["preview"] for row in read_jsonl(output)} == expected
-
-
-def test_revision_preview_uses_the_same_read_window_as_local_files(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- repo.mkdir()
- initialize_repo(repo)
- source = repo / "example.py"
- source.write_text("def first():\n pass\n")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
- source.write_text(
- "def first():\n pass\n#" + "x" * 80_000 + "\ndef outside_window():\n pass\n"
- )
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "long source")
- output = tmp_path / "preview.jsonl"
- run_repo_cli(
- "make-diff-rank-input",
- repo,
- output,
- "--base",
- base,
- "--head",
- "HEAD",
- )
- preview = read_jsonl(output)[0]["preview"]
- assert preview.startswith("def first():\n pass")
- assert "outside_window" not in preview
- assert len(preview.encode("utf-8")) <= 1024
-
-
-def test_local_diff_inventory_excludes_svn_but_revision_diff_retains_it(tmp_path: Path) -> None:
- repo = tmp_path / "repo"
- repo.mkdir()
- initialize_repo(repo)
- changes = {}
- sources = []
- for state in ("staged", "unstaged", "untracked", "deleted"):
- ordinary = [f"{state}.py", f"vendor/.svn-{state}.py"]
- sources.extend(ordinary)
- changes[state] = [*ordinary, f".svn/{state}", f"vendor/.svn/{state}"]
-
- def write_source(name: str, version: str) -> None:
- source = repo / name
- source.parent.mkdir(parents=True, exist_ok=True)
- source.write_text(f"{name}: {version}\n", encoding="utf-8")
-
- for state in ("staged", "unstaged", "deleted"):
- for name in changes[state]:
- write_source(name, "before")
- git(repo, "add", ".")
- git(repo, "commit", "-qm", "base")
- base = git(repo, "rev-parse", "HEAD")
-
- for state in ("staged", "unstaged", "untracked"):
- for name in changes[state]:
- write_source(name, "after")
- git(repo, "add", "--", *changes["staged"])
- for name in changes["deleted"]:
- (repo / name).unlink()
-
- def assert_inventories(mode: str, expected: list[str]) -> None:
- rank = tmp_path / f"{mode}.jsonl"
- run_repo_cli("make-diff-rank-input", repo, rank, "--base", base, "--mode", mode)
- assert [row["path"] for row in read_jsonl(rank)] == sorted(expected), mode
- inventory = tmp_path / f"{mode}.txt"
- subprocess.run(
- [
- sys.executable,
- str(SCRIPT.with_name("generate_in_scope_files.py")),
- "--repo",
- str(repo),
- "--scope",
- ".",
- "--out",
- str(inventory),
- "--diff-base",
- base,
- "--diff-mode",
- mode,
- ],
- check=True,
- capture_output=True,
- text=True,
- )
- assert inventory.read_text(encoding="utf-8").splitlines() == sorted(expected), mode
-
- assert_inventories("local-patch", sources)
- git(repo, "add", "-A")
- git(repo, "commit", "-qm", "change all four paths in each group")
- assert_inventories("revisions", [name for names in changes.values() for name in names])
diff --git a/plugins/codex-security/tests/test_git_inventory_regressions.py b/plugins/codex-security/tests/test_git_inventory_regressions.py
index 385d676b0..b70137d6b 100644
--- a/plugins/codex-security/tests/test_git_inventory_regressions.py
+++ b/plugins/codex-security/tests/test_git_inventory_regressions.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import argparse
import hashlib
import json
import os
@@ -229,36 +228,6 @@ def test_dirty_submodule_warning_preserves_the_changed_target_detail(tmp_path: P
assert "results were saved" in warning
-def test_committed_binary_detection_agrees_beyond_preview_window(tmp_path: Path) -> None:
- target = tmp_path / "target"
- initialize_git_repository(target)
- base = git(target, "rev-parse", "HEAD").decode()
- (target / "payload.php").write_bytes(b"<?php\n" + b" " * (70 * 1024) + b"\0payload")
- (target / "visible.py").write_text("value = 1\n")
- git(target, "add", ".")
- git(target, "commit", "-qm", "Add source and binary fixtures")
- inventory_path = tmp_path / "inventory.txt"
- rank_path = tmp_path / "rank.jsonl"
- load_script("generate_in_scope_files").generate_diff_in_scope_files(
- target, base, "HEAD", "revisions", inventory_path
- )
- load_script("generate_rank_input").make_diff_rank_input(
- argparse.Namespace(
- repo=str(target),
- base=base,
- head="HEAD",
- mode="revisions",
- out=str(rank_path),
- area="fixture",
- preview_bytes=1024,
- )
- )
- assert inventory_path.read_text().splitlines() == ["visible.py"]
- assert [json.loads(line)["path"] for line in rank_path.read_text().splitlines()] == [
- "visible.py"
- ]
-
-
def test_disabled_git_source_snapshot_does_not_inspect_refs(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -278,55 +247,3 @@ def test_disabled_git_source_snapshot_does_not_inspect_refs(
)["source"]
assert source["revision"] == "unversioned"
assert source["refsDigest"] == hashlib.sha256(b"").hexdigest()
-
-
[email protected]("mode", ["revisions", "local-patch"])
[email protected]("replacement", ["symlink", "gitlink"])
-def test_diff_inventories_exclude_non_file_type_changes(
- tmp_path: Path, mode: str, replacement: str
-) -> None:
- target = tmp_path / "target"
- initialize_git_repository(target)
- source = target / "replaced.py"
- source.write_text("old source\n")
- git(target, "add", ".")
- git(target, "commit", "-qm", "Add source fixture")
- base = git(target, "rev-parse", "HEAD").decode()
- source.unlink()
- if replacement == "symlink":
- try:
- source.symlink_to("README.md")
- except OSError:
- pytest.skip("symlinks are unavailable")
- git(target, "add", "--", source.name)
- else:
- # The diff consumes the gitlink's index mode, so it needs no child clone.
- source.mkdir()
- git(target, "update-index", "--add", "--cacheinfo", f"160000,{base},{source.name}")
- (target / ".gitmodules").write_text(
- f'[submodule "fixture"]\n\tpath = {source.name}\n\turl = ../fixture\n'
- )
- git(target, "add", ".gitmodules")
- (target / "visible.py").write_text("value = 1\n")
- git(target, "add", "visible.py")
- if mode == "revisions":
- git(target, "commit", "-qm", "Replace source type")
- inventory_path = tmp_path / "inventory.txt"
- rank_path = tmp_path / "rank.jsonl"
- load_script("generate_in_scope_files").generate_diff_in_scope_files(
- target, base, "HEAD", mode, inventory_path
- )
- load_script("generate_rank_input").make_diff_rank_input(
- argparse.Namespace(
- repo=str(target),
- base=base,
- head="HEAD",
- mode=mode,
- out=str(rank_path),
- area="fixture",
- preview_bytes=1024,
- )
- )
- expected = [".gitmodules", "visible.py"] if replacement == "gitlink" else ["visible.py"]
- assert inventory_path.read_text().splitlines() == expected
- assert [json.loads(line)["path"] for line in rank_path.read_text().splitlines()] == expected
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
deleted file mode 100644
index c84f53d53..000000000
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ /dev/null
@@ -1,297 +0,0 @@
-from __future__ import annotations
-
-import sys
-from io import BytesIO
-from pathlib import Path
-from unittest.mock import patch
-
-import pytest
-
-PLUGIN_ROOT = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(PLUGIN_ROOT / "scripts"))
-from rank_preview import DEFAULT_PREVIEW_BYTES, preview_for, preview_for_bytes
-
-
- params=[("utf-16-le", b"\xff\xfe"), ("utf-16-be", b"\xfe\xff")],
- ids=["utf16-le", "utf16-be"],
-)
-def utf16_encoding(request: pytest.FixtureRequest) -> tuple[str, bytes]:
- return request.param
-
-
-def test_preview_for_decodes_bom_marked_utf16(
- tmp_path: Path, utf16_encoding: tuple[str, bytes]
-) -> None:
- encoding, bom = utf16_encoding
- source = "Write-Output 'café 😀'\n"
- path = tmp_path / "source.ps1"
- data = bom + source.encode(encoding)
- path.write_bytes(data)
- expected = (source.strip(), False)
-
- assert preview_for(path, DEFAULT_PREVIEW_BYTES) == expected
- assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == expected
- assert preview_for_bytes(source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == expected
-
-
- "data",
- [
- b"header\0payload",
- b"\xff\xfe" + "text\0binary".encode("utf-16-le"),
- b"\xfe\xff" + "text\0binary".encode("utf-16-be"),
- "unmarked utf16".encode("utf-16-le"),
- ],
- ids=["generic-binary", "utf16-le-nul", "utf16-be-nul", "no-bom"],
-)
-def test_preview_for_rejects_binary_source_bytes(tmp_path: Path, data: bytes) -> None:
- path = tmp_path / "source.ps1"
- path.write_bytes(data)
-
- assert preview_for(path, DEFAULT_PREVIEW_BYTES) == ("", True)
- assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == ("", True)
-
-
-def test_preview_for_handles_utf16_surrogate_at_sample_boundary(
- tmp_path: Path, utf16_encoding: tuple[str, bytes]
-) -> None:
- encoding, bom = utf16_encoding
- source = "a" * 2046 + "😀\nWrite-Output 'done'\n"
- path = tmp_path / "source.ps1"
- data = bom + source.encode(encoding)
- path.write_bytes(data)
-
- assert preview_for(path, 8192) == (source.strip(), False)
- assert preview_for_bytes(data, 8192) == (source.strip(), False)
-
-
-def test_preview_for_bounds_utf16_source_at_incomplete_character(
- tmp_path: Path, utf16_encoding: tuple[str, bytes]
-) -> None:
- encoding, bom = utf16_encoding
- source = "a" * 32766 + "😀\nWrite-Output 'outside preview'\n"
- path = tmp_path / "source.ps1"
- path.write_bytes(bom + source.encode(encoding))
-
- assert preview_for(path, 128, max_read_bytes=64 * 1024) == ("a" * 128, False)
-
-
-def test_preview_for_ignores_incomplete_utf16_tail(
- tmp_path: Path, utf16_encoding: tuple[str, bytes]
-) -> None:
- encoding, bom = utf16_encoding
- source = "Write-Output 'café'"
- path = tmp_path / "source.ps1"
- data = bom + source.encode(encoding) + b"\0"
- path.write_bytes(data)
-
- assert preview_for(path, DEFAULT_PREVIEW_BYTES) == (source, False)
- assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == (source, False)
-
-
-class BudgetedReader(BytesIO):
- def __init__(self, data: bytes, limit: int) -> None:
- super().__init__(data)
- self.limit = limit
- self.bytes_read = 0
-
- def read(self, size: int = -1) -> bytes:
- assert 0 <= size <= self.limit - self.bytes_read
- data = super().read(size)
- self.bytes_read += len(data)
- return data
-
-
- ("filename", "nul_offset", "limit"),
- [("payload.bin", 0, 4096), ("payload.py", 4096, 64 * 1024)],
- ids=["initial-sample", "after-initial-sample"],
-)
-def test_preview_for_bounds_binary_reads(filename: str, nul_offset: int, limit: int) -> None:
- reader = BudgetedReader(b"x" * nul_offset + b"\0" + b"x" * limit, limit)
- with patch.object(Path, "open", return_value=reader):
- assert preview_for(Path(filename), DEFAULT_PREVIEW_BYTES) == ("", True)
- assert reader.bytes_read > nul_offset
-
-
[email protected]("max_read_bytes", [None, 1024, 4096, 4097, 128 * 1024])
-def test_preview_for_bounds_large_text_reads(max_read_bytes: int | None) -> None:
- limit = 64 * 1024 if max_read_bytes is None else max_read_bytes
- line = b"# source comment\n"
- reader = BudgetedReader(line * (limit // len(line) + 1), limit)
- with patch.object(Path, "open", return_value=reader):
- if max_read_bytes is None:
- preview, binary = preview_for(Path("large.py"), 128)
- else:
- preview, binary = preview_for(Path("large.py"), 128, max_read_bytes=max_read_bytes)
-
- assert reader.bytes_read == limit
- assert not binary
- assert preview
- assert len(preview.encode("utf-8")) <= 128
-
-
[email protected]("max_read_bytes", [0, -1])
-def test_preview_for_rejects_nonpositive_read_limit(max_read_bytes: int) -> None:
- with patch.object(Path, "open", side_effect=AssertionError("unexpected file read")):
- with pytest.raises(ValueError, match="max_read_bytes must be positive"):
- preview_for(Path("source.py"), 128, max_read_bytes=max_read_bytes)
-
-
- ("filename", "prefix", "suffix", "expected"),
- [
- ("source.py", b"def visible():\n value = (", b"1)\n", "def visible():\n value = ("),
- ("source.css", b"body { color: red; }\n\xf0\x9f", b"\x98\x80", "body { color: red; }"),
- ("source.css", b"body { color: red; }\n", b"\0binary", "body { color: red; }"),
- ],
-)
-def test_preview_for_uses_only_the_bounded_prefix(
- tmp_path: Path, filename: str, prefix: bytes, suffix: bytes, expected: str
-) -> None:
- source = tmp_path / filename
- source.write_bytes(prefix + suffix)
-
- assert preview_for(source, 128, max_read_bytes=len(prefix)) == (expected, False)
-
-
-def generate_preview(
- tmp_path: Path, filename: str, source: str, *, preview_bytes: int | None = None
-) -> str:
- source_path = tmp_path / filename
- source_path.parent.mkdir(parents=True, exist_ok=True)
- source_path.write_text(source, encoding="utf-8")
- preview, is_binary = preview_for(
- source_path,
- DEFAULT_PREVIEW_BYTES if preview_bytes is None else preview_bytes,
- )
- assert not is_binary
- return preview
-
-
-def test_small_source_is_complete(tmp_path: Path) -> None:
- source = (
- "\n\n# Source context\n"
- + "\n".join(f"setting_{index} = {index}" for index in range(24))
- + '\n\ndef handle(request):\n text = "two spaces"\n'
- + ' return subprocess.run(request.args["cmd"], shell=True)\n\n'
- )
- assert len(source.strip().encode("utf-8")) < DEFAULT_PREVIEW_BYTES
-
- preview = generate_preview(tmp_path, "entrypoint", source)
-
- assert preview == source.strip()
- assert preview_for_bytes(source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == (
- preview,
- False,
- )
-
-
[email protected]("large", [False, True], ids=["complete", "sampled"])
-def test_source_preview_preserves_retained_whitespace(tmp_path: Path, large: bool) -> None:
- retained = ' value = "two spaces\tand\u0085a separator" \n return value '
- source = "\n \t\n" + retained
- if large:
- source += "\n# " + "x" * DEFAULT_PREVIEW_BYTES
- source += "\n \t\n"
-
- preview = generate_preview(tmp_path, "source.py", source)
-
- if large:
- assert preview.startswith(retained + "\n")
- assert len(preview.encode("utf-8")) <= DEFAULT_PREVIEW_BYTES
- else:
- assert preview == retained
-
-
[email protected]("newline", ["\n", "\r\n", "\r"], ids=["lf", "crlf", "cr"])
-def test_source_preview_normalizes_newlines(tmp_path: Path, newline: str) -> None:
- expected = 'def handle():\n text = "two spaces"\n\n return text'
- data = (expected.replace("\n", newline) + newline).encode("utf-8")
- path = tmp_path / "source.py"
- path.write_bytes(data)
-
- assert preview_for(path, DEFAULT_PREVIEW_BYTES) == (expected, False)
- assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == (expected, False)
-
-
[email protected]("preview_bytes", [0, -1])
-def test_nonpositive_preview_budget_returns_no_source(preview_bytes: int) -> None:
- assert preview_for_bytes(b"value = 1\n", preview_bytes) == ("", False)
-
-
-def test_empty_source_preview(tmp_path: Path) -> None:
- assert generate_preview(tmp_path, "empty.py", "\n \t\n") == ""
-
-
-def test_large_source_preview_preserves_head_and_tail(tmp_path: Path) -> None:
- source = (
- "function first() { return count / total; }\n"
- + ("// " + "é" * 128 + "\n") * 100
- + "function last() {}\n"
- )
- path = tmp_path / "example.js"
- path.write_text(source, encoding="utf-8")
-
- preview, binary = preview_for(path, DEFAULT_PREVIEW_BYTES)
-
- assert binary is False
- assert len(preview.encode("utf-8")) <= DEFAULT_PREVIEW_BYTES
- assert "function first()" in preview
- assert "function last()" in preview
- assert "..." in preview
-
-
-def test_source_preview_ignores_utf8_bom(tmp_path: Path) -> None:
- source = b"def first():\n pass\n\ndef second():\n pass\n"
- path = tmp_path / "example.py"
- path.write_bytes(b"\xef\xbb\xbf" + source)
-
- expected = preview_for_bytes(source, DEFAULT_PREVIEW_BYTES)
- assert expected == (source.decode("utf-8").strip(), False)
- assert preview_for(path, DEFAULT_PREVIEW_BYTES) == expected
-
-
-def test_large_source_preview_samples_nonblank_lines(tmp_path: Path) -> None:
- source = "\n\n".join(f"line_{index:02d} {{ color: red; }}" for index in range(40))
-
- preview = generate_preview(tmp_path, "styles.css", source, preview_bytes=700)
-
- assert preview.splitlines() == [
- *(f"line_{index:02d} {{ color: red; }}" for index in range(12)),
- "...",
- *(
- f"line_{index:02d} {{ color: red; }}"
- for index in (12, 15, 18, 21, 24, 27, 30, 33, 36, 39)
- ),
- ]
-
-
-def test_preview_byte_budget_preserves_sampled_tail_and_valid_unicode(tmp_path: Path) -> None:
- source = "\n".join(f"line_{index:02d} {'😀' * 20}" for index in range(40))
-
- preview = generate_preview(tmp_path, "styles.css", source, preview_bytes=220)
-
- assert len(preview.encode("utf-8")) <= 220
- assert "..." in preview
- assert "line_39" in preview
-
-
-def test_literal_elision_line_respects_tiny_byte_budget(tmp_path: Path) -> None:
- preview = generate_preview(tmp_path, "styles.css", "...", preview_bytes=2)
-
- assert preview == ".."
-
-
[email protected](("prefix", "preview_bytes"), [("line", 25), ("😀", 30), (" line", 80)])
-def test_tiny_preview_budget_retains_source(
- tmp_path: Path, prefix: str, preview_bytes: int
-) -> None:
- source = "\n".join(f"{prefix}_{index:02d}" for index in range(40))
-
- preview = generate_preview(tmp_path, "source.txt", source, preview_bytes=preview_bytes)
-
- assert preview.startswith(f"{prefix}_00")
- assert source.startswith(preview)
- assert len(preview.encode("utf-8")) <= preview_bytes
diff --git a/plugins/codex-security/tests/test_windows_scope_streams.py b/plugins/codex-security/tests/test_windows_scope_streams.py
deleted file mode 100644
index d6bc968c2..000000000
--- a/plugins/codex-security/tests/test_windows_scope_streams.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from __future__ import annotations
-
-import runpy
-from pathlib import Path, PureWindowsPath
-from types import SimpleNamespace
-
-import pytest
-
-
-class WindowsScopePath(PureWindowsPath):
- def expanduser(self):
- return self
-
-
[email protected]("script", ["generate_in_scope_files.py", "generate_rank_input.py"])
-def test_scope_rejects_ntfs_streams_before_resolving_paths(monkeypatch, script: str) -> None:
- path = Path(__file__).resolve().parents[1] / "scripts" / script
- namespace = runpy.run_path(str(path), run_name="scope_stream_test")
- resolve_scope = namespace["resolve_scope"]
- stream_component = namespace["windows_stream_component"]
- monkeypatch.setitem(stream_component.__globals__, "os", SimpleNamespace(name="nt"))
- monkeypatch.setitem(resolve_scope.__globals__, "Path", WindowsScopePath)
- root = PureWindowsPath(r"C:\Repository")
- error = namespace.get("InventoryError", SystemExit)
- with pytest.raises(error, match="alternate data stream"):
- resolve_scope(root, "src/app.py:stream")
- assert namespace["windows_stream_component"](root / "src/app.py") is None
- monkeypatch.setitem(stream_component.__globals__, "os", SimpleNamespace(name="posix"))
- assert namespace["windows_stream_component"](Path("src/app.py:fixture")) is None
diff --git a/plugins/codex-security/tests/test_workbench_target.py b/plugins/codex-security/tests/test_workbench_target.py
index acf82a32a..d19094448 100644
--- a/plugins/codex-security/tests/test_workbench_target.py
+++ b/plugins/codex-security/tests/test_workbench_target.py
@@ -919,10 +919,6 @@ def test_git_blob_samples_match_full_file_classification_with_bounded_reads(
]
assert max(reads) == 64 * 1024
- from rank_preview import is_binary_file
-
- assert [is_binary_file(target / name) for name in contents] == [False, True, True, False, False]
-
@pytest.mark.parametrize(
"output",
@@ -1595,6 +1591,23 @@ def test_copy_retains_alias_rooted_gitlink_exclusions(
assert not (copied / "submodule").exists()
+def test_candidate_sources_disambiguate_revision_from_worktree_filename(tmp_path: Path) -> None:
+ target = tmp_path / "repository"
+ revision = initialize_git_repository(target)
+ (target / revision).write_text("synthetic revision-named file\n")
+ subprocess.run(["git", "rm", "-q", "README.md"], cwd=target, check=True)
+
+ sources = WORKBENCH_TARGET["candidate_source_lines"](
+ target,
+ {"kind": "working_tree", "baseRevision": revision, "headRevision": revision},
+ ["README.md"],
+ ["README.md"],
+ )
+
+ assert sources == {"README.md": {"path": "README.md", "lineCount": 1}}
+
+
@pytest.mark.parametrize("replacement", ["directory", "ignored_file", "link"])
@pytest.mark.parametrize("case_insensitive", [False, True])
def test_deleted_candidate_sources_keep_base_blob_when_path_is_recreated(
@@ -1625,9 +1638,12 @@ def test_deleted_candidate_sources_keep_base_blob_when_path_is_recreated(
)
else:
source.symlink_to(outside, target_is_directory=True)
- from generate_rank_input import git_changed_paths
-
- assert dict(git_changed_paths(target, revision, revision, "local-patch"))[source] == "D"
+ assert (
+ dict(WORKBENCH_TARGET["git_changed_paths"](target, revision, revision, "local-patch"))[
+ source
+ ]
+ == "D"
+ )
sources = WORKBENCH_TARGET["candidate_source_lines"](
target,
{"kind": "working_tree", "baseRevision": revision, "headRevision": revision},
diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts
index 13488260e..1a983ee7e 100644
--- a/sdk/typescript/src/api.ts
+++ b/sdk/typescript/src/api.ts
@@ -18,6 +18,7 @@ import {
readdir,
realpath,
rm,
+ stat,
writeFile,
} from "node:fs/promises";
import { randomUUID } from "node:crypto";
@@ -2731,7 +2732,7 @@ export class CodexSecurity {
}
checkOpen();
const basePrompt = scanPrompt(
- normalized,
+ await targetInstruction(normalized, shellPluginRoot),
mode,
skillName,
scanId,
@@ -5197,7 +5198,7 @@ function trustedAccessWarning(
}
function scanPrompt(
- target: NormalizedTarget,
+ targetDescription: string,
mode: ScanMode,
skillName: string,
scanId: string,
@@ -5276,8 +5277,8 @@ function scanPrompt(
: []),
]
: []),
- "Runtime paths are environment-backed; keep them quoted in POSIX shells and use the corresponding $env: names in PowerShell. Do not copy or reparse their values.",
- targetInstruction(target, python),
+ "Runtime paths are environment-backed; keep references quoted. Use $NAME in POSIX shells, $env:NAME in PowerShell expressions, and %NAME% inside CMD command strings. Do not copy or reparse their values.",
+ targetDescription,
...(mode === "deep"
? [
"Once the Deep Scan coordinator returns its aggregate, end this turn with the returned scanId and manifestPath. The SDK owns finalization and report generation after the turn ends.",
@@ -5301,23 +5302,40 @@ function skillNameFor(target: NormalizedTarget, mode: ScanMode): string {
return mode === "deep" ? "deep-security-scan" : "security-scan";
}
-function targetInstruction(target: NormalizedTarget, python: string): string {
+async function targetInstruction(
+ target: NormalizedTarget,
+ pluginRoot: string,
+): Promise<string> {
if (target.kind === "repository")
return "Scan target: the entire repository.";
if (target.kind === "paths") {
const helper = shellEnvironmentReference(
"CODEX_SECURITY_PLUGIN_ROOT",
- "/scripts/generate_rank_input.py",
+ "/scripts/launch_codex_security_mcp",
);
const scopes = shellEnvironmentReference(
"CODEX_SECURITY_TARGET_PATHS_FILE",
);
+ let legacyGenerator = false;
+ try {
+ legacyGenerator = (
+ await stat(join(pluginRoot, "scripts", "generate_rank_input.py"))
+ ).isFile();
+ } catch (error) {
+ const code = (error as NodeJS.ErrnoException).code;
+ if (code !== "ENOENT" && code !== "ENOTDIR") throw error;
+ }
+ const makeScope = legacyGenerator
+ ? `${pluginPythonCommand()} ${shellEnvironmentReference("CODEX_SECURITY_PLUGIN_ROOT", "/scripts/generate_rank_input.py")} make-repo-scope-input --repo ${shellEnvironmentReference("CODEX_SECURITY_REPOSITORY")} --scopes-file ${scopes} --out ${shellEnvironmentReference("CODEX_SECURITY_SCAN_DIR", "/scoped-source-input.jsonl")}`
+ : process.platform === "win32"
+ ? String.raw`cmd.exe /d /v:off /s /c '""%CODEX_SECURITY_PLUGIN_ROOT%\scripts\launch_codex_security_mcp.cmd" --helper make-repo-scope-input --repo "%CODEX_SECURITY_REPOSITORY%\." --scopes-file "%CODEX_SECURITY_TARGET_PATHS_FILE%" --out "%CODEX_SECURITY_SCAN_DIR%\scoped-source-input.jsonl""'`
+ : `${helper} --helper make-repo-scope-input --repo ${shellEnvironmentReference("CODEX_SECURITY_REPOSITORY")} --scopes-file ${scopes} --out ${shellEnvironmentReference("CODEX_SECURITY_SCAN_DIR", "/scoped-source-input.jsonl")}`;
// CMD expands the environment references once, preserving percent signs in path values.
const bindScopes =
process.platform === "win32"
? String.raw`cmd.exe /d /v:off /s /c '""%CODEX_SECURITY_PLUGIN_ROOT%\scripts\launch_codex_security_mcp.cmd" --helper bind-repo-scopes --scopes-file "%CODEX_SECURITY_TARGET_PATHS_FILE%" --manifest "%CODEX_SECURITY_SCAN_DIR%\scan-manifest.json" --coverage "%CODEX_SECURITY_SCAN_DIR%\coverage.json""'`
: `${shellEnvironmentReference("CODEX_SECURITY_PLUGIN_ROOT", "/scripts/launch_codex_security_mcp")} --helper bind-repo-scopes --scopes-file ${scopes} --manifest ${shellEnvironmentReference("CODEX_SECURITY_SCAN_DIR", "/scan-manifest.json")} --coverage ${shellEnvironmentReference("CODEX_SECURITY_SCAN_DIR", "/coverage.json")}`;
- return `Scan target paths: resolve every requested file and all non-ignored descendants of requested directories using ${python} ${helper} make-repo-scope-input --repo ${shellEnvironmentReference("CODEX_SECURITY_REPOSITORY")} --scopes-file ${scopes} --out ${shellEnvironmentReference("CODEX_SECURITY_SCAN_DIR", "/scoped-source-input.jsonl")}. Before finalization, preserve every requested scope with ${bindScopes}. Do not print, evaluate, or modify the target-paths file.`;
+ return `Scan target paths: resolve every requested file and all non-ignored descendants of requested directories using ${makeScope}. Before finalization, preserve every requested scope with ${bindScopes}. Do not print, evaluate, or modify the target-paths file.`;
}
if (target.kind === "refs") {
return `Scan target: Git diff from ${target.base} to ${target.head}.`;
diff --git a/sdk/typescript/src/custom-validation-prompt.ts b/sdk/typescript/src/custom-validation-prompt.ts
index 924166c59..fdb37e4ec 100644
--- a/sdk/typescript/src/custom-validation-prompt.ts
+++ b/sdk/typescript/src/custom-validation-prompt.ts
@@ -11,7 +11,7 @@ const SOURCES = {
"references/core-scan.md":
"e0c0e756f6d847ff90b10ed6ad6de37fd827cab18fb1eec25bbb8a25f9a0e61c",
"skills/security-scan/SKILL.md":
- "c07fc3493b0b315541650892e85c130f90831e52401249f6a18a35b380950a74",
+ "f236ddc65208fa4a07786923432111a03104286a9a6d2f27b4857a6e9e2a212b",
"skills/security-diff-scan/SKILL.md":
"8a2890f79792dd441849fa4a30a6f2f5469ca1b14224d25ff83637ee7fd23cf5",
} as const;
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index 00de9e092..b4c541273 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -6894,12 +6894,15 @@ describe("CodexSecurity orchestration", () => {
);
const helper = shellEnvironmentReference(
"CODEX_SECURITY_PLUGIN_ROOT",
- "/scripts/generate_rank_input.py",
+ "/scripts/launch_codex_security_mcp",
);
const scopes = shellEnvironmentReference(
"CODEX_SECURITY_TARGET_PATHS_FILE",
);
- const makeScopeCommand = `${pythonCommand} ${helper} make-repo-scope-input --repo ${shellEnvironmentReference("CODEX_SECURITY_REPOSITORY")} --scopes-file ${scopes} --out ${shellEnvironmentReference("CODEX_SECURITY_SCAN_DIR", "/scoped-source-input.jsonl")}`;
+ const makeScopeCommand =
+ process.platform === "win32"
+ ? String.raw`cmd.exe /d /v:off /s /c '""%CODEX_SECURITY_PLUGIN_ROOT%\scripts\launch_codex_security_mcp.cmd" --helper make-repo-scope-input --repo "%CODEX_SECURITY_REPOSITORY%\." --scopes-file "%CODEX_SECURITY_TARGET_PATHS_FILE%" --out "%CODEX_SECURITY_SCAN_DIR%\scoped-source-input.jsonl""'`
+ : `${helper} --helper make-repo-scope-input --repo ${shellEnvironmentReference("CODEX_SECURITY_REPOSITORY")} --scopes-file ${scopes} --out ${shellEnvironmentReference("CODEX_SECURITY_SCAN_DIR", "/scoped-source-input.jsonl")}`;
const bindScopeCommand =
process.platform === "win32"
? String.raw`cmd.exe /d /v:off /s /c '""%CODEX_SECURITY_PLUGIN_ROOT%\scripts\launch_codex_security_mcp.cmd" --helper bind-repo-scopes --scopes-file "%CODEX_SECURITY_TARGET_PATHS_FILE%" --manifest "%CODEX_SECURITY_SCAN_DIR%\scan-manifest.json" --coverage "%CODEX_SECURITY_SCAN_DIR%\coverage.json""'`
@@ -6943,10 +6946,11 @@ describe("CodexSecurity orchestration", () => {
`${repository}\0${scanDir}\0${python}\0${serializedPaths}\n`,
);
}
- const interpreter = pythonExecutable(false);
- expect(interpreter).not.toBeNull();
const scopedSourceInput = join(scanDir, "scoped-source-input.jsonl");
- const runScopedHelper = (command: string): void => {
+ const runScopedHelper = (
+ command: string,
+ overrides: NodeJS.ProcessEnv = {},
+ ): void => {
const shell =
process.platform === "win32" ? Bun.which("powershell.exe") : "/bin/sh";
expect(shell).not.toBeNull();
@@ -6960,11 +6964,10 @@ describe("CodexSecurity orchestration", () => {
env: {
...process.env,
...environment,
- PYTHON: interpreter!,
- PYTHONDONTWRITEBYTECODE: "1",
CODEX_MCP_NODE_PATH: Bun.which("node")!,
PATH_LITERAL: "expanded-wrong-directory",
CODEX_SECURITY_TARGET_PATHS_FILE: capturedTargetPathsFile,
+ ...overrides,
},
stdio: "pipe",
},
@@ -6994,9 +6997,128 @@ describe("CodexSecurity orchestration", () => {
expect(JSON.parse(await readFile(coverage, "utf8")).includePaths).toEqual(
paths,
);
+ if (process.platform === "win32") {
+ const driveRoot = win32.parse(repository).root;
+ const absoluteScopes = join(root, "absolute-scopes.json");
+ await writeFile(
+ absoluteScopes,
+ JSON.stringify(paths.map((value) => join(repository, value))),
+ );
+ runScopedHelper(makeScopeCommand, {
+ CODEX_SECURITY_REPOSITORY: driveRoot,
+ CODEX_SECURITY_TARGET_PATHS_FILE: absoluteScopes,
+ });
+ expect(
+ (await readFile(scopedSourceInput, "utf8"))
+ .trimEnd()
+ .split("\n")
+ .map((row) => JSON.parse(row).path),
+ ).toEqual(
+ paths
+ .map((value) =>
+ win32
+ .relative(driveRoot, join(repository, value))
+ .replaceAll("\\", "/"),
+ )
+ .sort(),
+ );
+ }
await client.close();
});
+ test("uses an existing Python scope generator from a custom plugin", async () => {
+ const root = await temporaryDirectory();
+ const repository = join(root, "repository");
+ const pluginRoot = join(root, "legacy plugin");
+ const codexHome = join(root, "codex-home");
+ const scanDir = join(root, "scan");
+ const python = pythonExecutable()!;
+ await mkdir(repository);
+ await mkdir(codexHome);
+ await mkdir(scanDir, { mode: 0o700 });
+ await mkdir(join(pluginRoot, "skills", "security-scan"), {
+ recursive: true,
+ });
+ await mkdir(join(pluginRoot, "scripts"));
+ await writeFile(join(repository, "source [1].ts"), "export {};\n");
+ await writeFile(
+ join(pluginRoot, "skills", "security-scan", "SKILL.md"),
+ "Review the requested source.\n",
+ );
+ await writeFile(
+ join(pluginRoot, "scripts", "generate_rank_input.py"),
+ [
+ "import argparse, json",
+ "from pathlib import Path",
+ "parser = argparse.ArgumentParser()",
+ "parser.add_argument('command', choices=['make-repo-scope-input'])",
+ "for name in ('repo', 'scopes-file', 'out'): parser.add_argument('--' + name, required=True)",
+ "args = parser.parse_args()",
+ "scopes = json.loads(Path(args.scopes_file).read_text())",
+ "Path(args.out).write_text(''.join(json.dumps({'path': scope}) + '\\n' for scope in scopes))",
+ ].join("\n"),
+ );
+ const client = new TestClient(
+ { pluginPath: pluginRoot },
+ {
+ prepareRuntime: async () => {
+ const runtime = preparedRuntime(codexHome);
+ return {
+ ...runtime,
+ plugin: {
+ ...runtime.plugin,
+ pluginRoot,
+ installedRoot: pluginRoot,
+ },
+ };
+ },
+ resolvePluginPython: async () => python,
+ prepareOutputDir: async () => scanDir,
+ repositoryRevision: async () => "deadbeef",
+ createCodex: (options: CodexOptions) => ({
+ startThread: () => ({
+ id: null,
+ async runStreamed(prompt: string) {
+ const command =
+ /Scan target paths:.* using (.+)\. Before finalization/u.exec(
+ prompt,
+ )?.[1];
+ expect(command).toContain("/scripts/generate_rank_input.py");
+ expect(command).not.toContain("--helper make-repo-scope-input");
+ const shell =
+ process.platform === "win32"
+ ? Bun.which("powershell.exe")!
+ : "/bin/sh";
+ execFileSync(
+ shell,
+ process.platform === "win32"
+ ? ["-NoProfile", "-NonInteractive", "-Command", command!]
+ : ["-c", command!],
+ { env: { ...process.env, ...options.env }, stdio: "pipe" },
+ );
+ expect(
+ JSON.parse(
+ await readFile(
+ join(scanDir, "scoped-source-input.jsonl"),
+ "utf8",
+ ),
+ ),
+ ).toEqual({ path: "source [1].ts" });
+ throw new Error("legacy scope prepared");
+ },
+ }),
+ }),
+ },
+ );
+ try {
+ await expect(
+ client.run(repository, { target: ["source [1].ts"] }),
+ ).rejects.toThrow("legacy scope prepared");
+ } finally {
+ await client.close();
+ }
+ });
+
test("keeps requested source paths without ranking or ignored directory files", async () => {
const root = await temporaryDirectory();
const repository = join(root, "repository");
@@ -7005,9 +7127,6 @@ describe("CodexSecurity orchestration", () => {
const vendored = join(source, "vendor");
const scopes = join(root, "scopes.json");
const output = join(root, "scoped-source-input.jsonl");
- const interpreter = pythonExecutable(false);
- expect(interpreter).not.toBeNull();
-
await mkdir(join(source, "tests"), { recursive: true });
await mkdir(join(source, "examples"));
await mkdir(ignored);
@@ -7042,10 +7161,9 @@ describe("CodexSecurity orchestration", () => {
const enumerate = async (requested: string[]) => {
await writeFile(scopes, JSON.stringify(requested));
execFileSync(
- interpreter!,
+ nodeCommand().command,
[
- "-B",
- join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
"make-repo-scope-input",
"--repo",
repository,
diff --git a/sdk/typescript/tests-ts/bind-repo-scopes.test.ts b/sdk/typescript/tests-ts/bind-repo-scopes.test.ts
index 1075691da..ed04d5682 100644
--- a/sdk/typescript/tests-ts/bind-repo-scopes.test.ts
+++ b/sdk/typescript/tests-ts/bind-repo-scopes.test.ts
@@ -301,7 +301,7 @@ for (const location of ["absolute", "relative", "unc"] as const)
process.platform !== "win32" ||
(location === "unc" && !hasWindowsLoopbackShare),
)(
- `documented PowerShell scope binding preserves ${location} literal input and artifact paths`,
+ `documented PowerShell scope generation and binding preserve ${location} literal paths`,
async () => {
const requested = ["caf\u00e9/\u96ea.py", "src", "src"];
const f = fixture(requested);
@@ -321,6 +321,11 @@ for (const location of ["absolute", "relative", "unc"] as const)
const launcher = windowsHelperFixture(f.root, {
CODEX_SECURITY_TARGET_PATHS_FILE: argument(scopes),
});
+ const repository = join(f.root, "repository %USERNAME% !EXPAND! café's");
+ mkdirSync(join(repository, "café"), { recursive: true });
+ mkdirSync(join(repository, "src"));
+ writeFileSync(join(repository, "café", "雪.py"), "source\n");
+ writeFileSync(join(repository, "src", "source.py"), "source\n");
const scanDir = join(f.root, "scan %USERNAME% !EXPAND! \u96ea's");
const expandedScanDir = scanDir.replace("%USERNAME%", "expanded-user");
for (const directory of [scanDir, expandedScanDir]) mkdirSync(directory);
@@ -329,6 +334,27 @@ for (const location of ["absolute", "relative", "unc"] as const)
"coverage.json": readFileSync(f.coverage),
};
for (const powershell of launcher.powershells) {
+ const generated = await launcher.run(
+ powershell,
+ "skills/security-scan/SKILL.md",
+ {
+ "<plugin_dir>": argument(launcher.plugin),
+ "<repo_root>": argument(repository),
+ "<scan_dir>": argument(scanDir),
+ },
+ undefined,
+ workingDirectory,
+ undefined,
+ 1,
+ );
+ expect(generated.status, generated.diagnostics).toBe(0);
+ expect(
+ readFileSync(join(scanDir, "scoped-source-input.jsonl"), "utf8")
+ .trim()
+ .split(/\r?\n/u)
+ .map((line) => JSON.parse(line).path),
+ generated.diagnostics,
+ ).toEqual(["café/雪.py", "src/source.py"]);
for (const [name, bytes] of Object.entries(initial)) {
writeFileSync(join(scanDir, name), bytes);
writeFileSync(join(expandedScanDir, name), bytes);
diff --git a/sdk/typescript/tests-ts/compact-diff-scan.test.ts b/sdk/typescript/tests-ts/compact-diff-scan.test.ts
index b337a7812..f53674940 100644
--- a/sdk/typescript/tests-ts/compact-diff-scan.test.ts
+++ b/sdk/typescript/tests-ts/compact-diff-scan.test.ts
@@ -1,3 +1,4 @@
+import { nodeCommand } from "./support/shell.js";
import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js";
import { git } from "./git-fixture.js";
import { startMcpClient } from "../../../plugins/codex-security/mcp-app/tests/support/mcp-client.js";
@@ -25,7 +26,7 @@ function createRepository(): { root: string; repository: string } {
return { root, repository };
}
-function python(script: string, ...args: string[]) {
+function pythonHelper(script: string, ...args: string[]) {
const command =
Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py");
expect(command).not.toBeNull();
@@ -36,6 +37,18 @@ function python(script: string, ...args: string[]) {
);
}
+function inventoryHelper(script: string, ...args: string[]) {
+ return spawnSync(
+ nodeCommand().command,
+ [
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ ...(script === "generate-in-scope-files" ? [script] : []),
+ ...args,
+ ],
+ { encoding: "utf8" },
+ );
+}
+
function candidate(path: string): JsonObject {
return {
cwe_ids: [],
@@ -48,7 +61,7 @@ function candidate(path: string): JsonObject {
async function startMcp(root: string) {
const { request, close } = await startMcpClient(
{
- command: process.execPath,
+ command: nodeCommand().command,
args: [join(PLUGIN_ROOT, "mcp", "server.mjs"), "--stdio"],
env: {
...process.env,
@@ -94,8 +107,8 @@ describe("compact diff scan", () => {
git(repository, "checkout", "--detach", base);
const output = join(root, "rank-input.jsonl");
- const result = python(
- "generate_rank_input.py",
+ const result = inventoryHelper(
+ "rank-input",
"make-diff-rank-input",
"--repo",
repository,
@@ -139,8 +152,8 @@ describe("compact diff scan", () => {
writeSource(repository, "src/handler.py", Buffer.from([0, 255, 1]));
const output = join(root, "in-scope.txt");
- const result = python(
- "generate_in_scope_files.py",
+ const result = inventoryHelper(
+ "generate-in-scope-files",
"--repo",
repository,
"--scope",
@@ -193,15 +206,11 @@ describe("compact diff scan", () => {
const head = git(repository, "rev-parse", "HEAD");
const output = join(root, "in-scope.txt");
- const executable = Bun.which("python3") ?? Bun.which("python");
- expect(executable).not.toBeNull();
const result = spawnSync(
- executable!,
+ nodeCommand().command,
[
- "-B",
- "-c",
- "import locale, runpy, sys; locale.setlocale(locale.LC_CTYPE, 'C'); runpy.run_path(sys.argv.pop(1), run_name='__main__')",
- join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"),
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "generate-in-scope-files",
"--repo",
repository,
"--scope",
@@ -235,8 +244,8 @@ describe("compact diff scan", () => {
writeSource(repository, "src/binary.py", Buffer.from([0, 255, 1]));
const output = join(root, "in-scope.txt");
- const result = python(
- "generate_in_scope_files.py",
+ const result = inventoryHelper(
+ "generate-in-scope-files",
"--repo",
repository,
"--scope",
@@ -257,8 +266,8 @@ describe("compact diff scan", () => {
]);
const reviewOutput = join(root, "rank-input.jsonl");
- const review = python(
- "generate_rank_input.py",
+ const review = inventoryHelper(
+ "rank-input",
"make-diff-rank-input",
"--repo",
repository,
@@ -790,7 +799,7 @@ describe("compact diff scan", () => {
] as const) {
writeFileSync(join(terminalDir, name), JSON.stringify(document));
}
- const finalized = python(
+ const finalized = pythonHelper(
"finalize_scan_contract.py",
"--scan-dir",
terminalDir,
@@ -798,7 +807,7 @@ describe("compact diff scan", () => {
repository,
);
expect(finalized.status, finalized.stderr).toBe(0);
- const validated = python(
+ const validated = pythonHelper(
"validate_scan_contract.py",
"--scan-dir",
terminalDir,
diff --git a/sdk/typescript/tests-ts/component-scan.test.ts b/sdk/typescript/tests-ts/component-scan.test.ts
index 1dda7a3ac..dff809345 100644
--- a/sdk/typescript/tests-ts/component-scan.test.ts
+++ b/sdk/typescript/tests-ts/component-scan.test.ts
@@ -233,6 +233,16 @@ async function scopedInventory(paths: Fixture, scope: string) {
const scopesFile = join(paths.root, "scopes.json");
const output = join(paths.root, "scoped-source-input.jsonl");
await writeFile(scopesFile, JSON.stringify([scope]));
+ execFileSync(nodeCommand().command, [
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "make-repo-scope-input",
+ "--repo",
+ paths.repository,
+ "--scopes-file",
+ scopesFile,
+ "--out",
+ output,
+ ]);
const stdout = execFileSync(
python!,
[
@@ -245,9 +255,7 @@ async function scopedInventory(paths: Fixture, scope: string) {
"from pathlib import Path",
"sys.path.insert(0, sys.argv[1])",
"import workbench_target as target",
- "from generate_rank_input import make_repo_scope_input",
"repo, scope, scopes, output = sys.argv[2:]",
- "make_repo_scope_input(Namespace(repo=repo, scopes_file=scopes, out=output))",
"rows = [json.loads(line)['path'] for line in Path(output).read_text().splitlines()]",
"count = target.directory_snapshot_regular_file_count((Path(repo) / scope).resolve())",
"print(json.dumps({'paths': rows, 'count': count}))",
diff --git a/sdk/typescript/tests-ts/config.test.ts b/sdk/typescript/tests-ts/config.test.ts
index 126f5c7cd..0bdbd0d29 100644
--- a/sdk/typescript/tests-ts/config.test.ts
+++ b/sdk/typescript/tests-ts/config.test.ts
@@ -1,3 +1,4 @@
+import { nodeCommand } from "./support/shell.js";
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { afterEach, describe, expect, test } from "bun:test";
@@ -656,9 +657,6 @@ describe("Codex configuration", () => {
await mkdir(repository);
await writeFile(join(repository, "fixture.txt"), "synthetic source\n");
const inventory = join(workspace, "in-scope-files.txt");
- const python =
- Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py");
- expect(python).not.toBeNull();
const helper = runPinnedCodex(
codexHome,
[
@@ -669,8 +667,9 @@ describe("Codex configuration", () => {
"codex_security_scan",
"--cd",
workspace,
- python!,
- join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"),
+ nodeCommand().command,
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "generate-in-scope-files",
"--repo",
repository,
"--scope",
diff --git a/sdk/typescript/tests-ts/diff-rank-input.test.ts b/sdk/typescript/tests-ts/diff-rank-input.test.ts
index 5ea3f07ec..be967dd03 100644
--- a/sdk/typescript/tests-ts/diff-rank-input.test.ts
+++ b/sdk/typescript/tests-ts/diff-rank-input.test.ts
@@ -1,6 +1,8 @@
import { runNodePython } from "./support/python-probe.js";
-import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js";
import { pythonExecutable } from "./support/python.js";
+import { nodeCommand } from "./support/shell.js";
+import { windowsHelperFixture } from "./windows-helper-command.js";
+import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js";
import { git } from "./git-fixture.js";
import { spawnSync } from "node:child_process";
import {
@@ -65,14 +67,11 @@ test("diff previews stay inside the selected repository", () => {
rmSync(nested, { recursive: true });
symlinkSync(externalFixture, nested, "junction");
- const python = pythonExecutable();
- expect(python).not.toBeNull();
const output = join(root, "rank-input.jsonl");
const result = spawnSync(
- python!,
+ nodeCommand().command,
[
- "-B",
- join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
"make-diff-rank-input",
"--repo",
repository,
@@ -136,21 +135,23 @@ test("preserves Unicode Git paths and legacy-encoded commit metadata", () => {
);
const legacyHead = git(repository, "rev-parse", "HEAD");
- const python = pythonExecutable();
- expect(python).not.toBeNull();
const output = join(root, "rank-input.jsonl");
- const rank = runNodePython(python!, [
- join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),
- "make-diff-rank-input",
- "--repo",
- repository,
- "--base",
- base,
- "--head",
- head,
- "--out",
- output,
- ]);
+ const rank = spawnSync(
+ nodeCommand().command,
+ [
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "make-diff-rank-input",
+ "--repo",
+ repository,
+ "--base",
+ base,
+ "--head",
+ head,
+ "--out",
+ output,
+ ],
+ { encoding: "utf8" },
+ );
const probeSource = [
"import json, pathlib, sys",
"sys.path.insert(0, sys.argv[1])",
@@ -162,6 +163,8 @@ test("preserves Unicode Git paths and legacy-encoded commit metadata", () => {
"diff = db.require_diff_target(repo, 'commit', None, sys.argv[3], None)",
"print(json.dumps({'root': str(root), 'pathspec': pathspec, 'subject': metadata['commitSubject'], 'diff': diff}))",
].join("\n");
+ const python = pythonExecutable();
+ expect(python).not.toBeNull();
const probe = runNodePython(python!, [
"-c",
probeSource,
@@ -222,9 +225,7 @@ testPosix(
writeFileSync(ripgrep, "#!/bin/sh\nexit 0\n");
chmodSync(ripgrep, 0o700);
- const python = pythonExecutable();
const hostGit = Bun.which("git");
- expect(python).not.toBeNull();
expect(hostGit).not.toBeNull();
const trustedGit = join(
realpathSync(dirname(hostGit!)),
@@ -232,10 +233,11 @@ testPosix(
);
const output = join(root, "output");
const run = (script: string, args: string[], binding = trustedGit) =>
- runNodePython(
- python!,
+ spawnSync(
+ nodeCommand().command,
[
- join(PLUGIN_ROOT, "scripts", script),
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ ...(script === "generate-in-scope-files" ? [script] : []),
...args,
"--repo",
repository,
@@ -243,6 +245,7 @@ testPosix(
output,
],
{
+ encoding: "utf8",
env: {
...process.env,
PATH: `${externalBin}:${process.env["PATH"] ?? ""}`,
@@ -258,16 +261,16 @@ testPosix(
"--head",
head,
];
- const unavailable = run("generate_rank_input.py", rankArguments, "");
+ const unavailable = run("rank-input", rankArguments, "");
expect(unavailable.status).not.toBe(0);
- const rejected = run("generate_rank_input.py", rankArguments, shim);
+ const rejected = run("rank-input", rankArguments, shim);
expect(rejected.status).not.toBe(0);
expect(rejected.stderr).toContain("outside the protected repository");
- const rank = run("generate_rank_input.py", rankArguments);
+ const rank = run("rank-input", rankArguments);
expect(rank.status, rank.stderr).toBe(0);
for (const diff of [true, false]) {
- const inventory = run("generate_in_scope_files.py", [
+ const inventory = run("generate-in-scope-files", [
"--scope",
".",
...(diff ? ["--diff-base", base, "--diff-head", head] : []),
@@ -280,3 +283,66 @@ testPosix(
expect(existsSync(marker)).toBe(false);
},
);
+
+for (const mode of ["revisions", "local-patch"] as const)
+ test.skipIf(process.platform !== "win32")(
+ `documented Windows ${mode} ranking command preserves literal paths with resolved revisions`,
+ async () => {
+ const root = temporaryRoots.create("codex-security-diff-docs-");
+ const repository = join(root, "repository %USERNAME% !EXPAND! 雪's");
+ const discovery = join(root, "discovery %USERNAME% !EXPAND! 雪's");
+ mkdirSync(repository);
+ mkdirSync(discovery);
+ git(repository, "init", "-q");
+ const source = join(repository, "source.py");
+ writeFileSync(source, "before\n");
+ git(repository, "add", ".");
+ git(repository, "commit", "-qm", "Fixture base");
+ const base = "fixture-%USERNAME%-!EXPAND!-é's";
+ git(repository, "branch", base);
+ writeFileSync(source, "after\n");
+ if (mode === "revisions") {
+ git(repository, "add", ".");
+ git(repository, "commit", "-qm", "Fixture head");
+ }
+ const launcher = windowsHelperFixture(root, {
+ CODEX_SECURITY_GIT: Bun.which("git")!,
+ });
+ for (const powershell of launcher.powershells) {
+ const result = await launcher.run(
+ powershell,
+ "skills/security-scan/references/scan-artifacts-and-ledger.md",
+ {
+ "<plugin_dir>": launcher.plugin,
+ "<repo_root>": repository,
+ "<discovery_dir>": discovery,
+ "<resolved_base_object_id>": git(
+ repository,
+ "rev-parse",
+ "--verify",
+ "--end-of-options",
+ `${base}^{commit}`,
+ ),
+ "<resolved_head_object_id>": git(
+ repository,
+ "rev-parse",
+ "--verify",
+ "--end-of-options",
+ "HEAD^{commit}",
+ ),
+ },
+ undefined,
+ root,
+ undefined,
+ mode === "revisions" ? 1 : 2,
+ );
+ expect(result.status, result.diagnostics).toBe(0);
+ expect(result.stdout, result.diagnostics).not.toContain(
+ "expanded-plugin-used",
+ );
+ expect(
+ JSON.parse(readFileSync(join(discovery, "rank_input.jsonl"), "utf8")),
+ ).toMatchObject({ path: "source.py", preview: "after" });
+ }
+ },
+ );
diff --git a/sdk/typescript/tests-ts/inventory-input.test.ts b/sdk/typescript/tests-ts/inventory-input.test.ts
new file mode 100644
index 000000000..23808db2e
--- /dev/null
+++ b/sdk/typescript/tests-ts/inventory-input.test.ts
@@ -0,0 +1,2518 @@
+import { afterEach, expect, test } from "bun:test";
+import { spawnSync } from "node:child_process";
+import {
+ chmodSync,
+ copyFileSync,
+ mkdirSync,
+ readFileSync,
+ readdirSync,
+ renameSync,
+ rmSync,
+ symlinkSync,
+ statSync,
+ writeFileSync,
+} from "node:fs";
+import { delimiter, dirname, join, parse, relative, sep } from "node:path";
+import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js";
+import { nodeCommand } from "./support/shell.js";
+import { git } from "./git-fixture.js";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+import { resolveCodexCommand } from "../src/runtime.js";
+import { bundledCodexSdkEnvironment } from "../src/codex-sdk-environment.js";
+
+const temporary = createTemporaryDirectoriesSync(true);
+const node = nodeCommand().command;
+// APFS requires valid UTF-8 names; Linux also permits undecodable bytes.
+const pathNameBytes =
+ process.platform === "darwin" ? Buffer.from("雪") : Buffer.from([0xff]);
+afterEach(temporary.cleanup);
+
+function fixture() {
+ const root = temporary.create("codex-security-inventory-"),
+ repo = join(root, "repository"),
+ out = join(root, "output");
+ mkdirSync(repo);
+ git(repo, "init", "-q");
+ const toolEnvironment = bundledCodexSdkEnvironment(
+ resolveCodexCommand({}).command,
+ Object.fromEntries(
+ Object.entries(process.env).filter(
+ (entry): entry is [string, string] => entry[1] !== undefined,
+ ),
+ ),
+ );
+ const write = (path: string, data: string | Buffer = "value = 1\n") => {
+ const file = join(repo, path);
+ mkdirSync(dirname(file), { recursive: true });
+ writeFileSync(file, data);
+ return file;
+ };
+ const run = (
+ command: string,
+ args: string[] = [],
+ env: NodeJS.ProcessEnv = toolEnvironment,
+ ) => {
+ const arguments_ = [
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ command,
+ "--repo",
+ repo,
+ "--out",
+ out,
+ ...args,
+ ];
+ const home = env["HOME"];
+ if (process.platform === "win32" && home && !home.isWellFormed()) {
+ const binding = join(
+ PLUGIN_ROOT,
+ "mcp",
+ "native",
+ `win32-${process.arch}`,
+ "windows.node",
+ );
+ const check = join(root, "raw-home-check.cjs");
+ writeFileSync(
+ check,
+ `require("node:assert/strict").equal(require(${JSON.stringify(binding)}).windowsEnvironment(Buffer.from("HOME", "utf16le"))?.toString("utf16le"), ${JSON.stringify(home)});`,
+ );
+ // The native launch preserves HOME before the helper reads its environment.
+ const script = `
+const native = require(${JSON.stringify(binding)});
+const wide = value => Buffer.from(value, "utf16le");
+const result = native.runWindowsProcess(wide(process.execPath), ["--require", ${JSON.stringify(check)}, ...process.argv.slice(1)].map(wide), undefined, [{ name: wide("HOME"), value: wide(${JSON.stringify(home)}) }]);
+if (result.error) throw Object.assign(new Error(result.message), { errno: result.error });
+process.exitCode = result.status;
+`;
+ return spawnSync(node, ["-e", script, ...arguments_], {
+ env,
+ encoding: "utf8",
+ });
+ }
+ return spawnSync(node, arguments_, { env, encoding: "utf8" });
+ };
+ const success = (command: string, args: string[] = []) => {
+ const result = run(command, args);
+ expect(result.status, result.stderr).toBe(0);
+ return readFileSync(out, "utf8");
+ };
+ const scopeArgs = (scope = ".") => {
+ const file = join(root, "requested-scopes.json");
+ writeFileSync(file, JSON.stringify([scope]));
+ return ["--scopes-file", file];
+ };
+ const rows = (command = "make-repo-scope-input", args?: string[]) =>
+ success(command, args ?? scopeArgs())
+ .split(/\r?\n/u)
+ .filter(Boolean)
+ .map(
+ (line) =>
+ JSON.parse(line) as { path: string; area?: string; preview?: string },
+ );
+ const localRows = (args: string[] = []) =>
+ rows("make-diff-rank-input", [
+ "--base",
+ git(repo, "hash-object", "-w", "-t", "tree", "--stdin"),
+ "--mode",
+ "local-patch",
+ ...args,
+ ]);
+ const commit = () => {
+ git(repo, "add", ".");
+ git(repo, "commit", "-qm", "Fixture revision");
+ return git(repo, "rev-parse", "HEAD");
+ };
+ return {
+ root,
+ repo,
+ out,
+ write,
+ run,
+ success,
+ rows,
+ localRows,
+ scopeArgs,
+ commit,
+ toolEnvironment,
+ };
+}
+
+test("committed binary detection agrees beyond the preview window", () => {
+ const f = fixture();
+ f.write("existing.py");
+ const base = f.commit();
+ f.write(
+ "payload.php",
+ Buffer.concat([
+ Buffer.from("<?php\n"),
+ Buffer.alloc(70 * 1024, 32),
+ Buffer.from("\0payload"),
+ ]),
+ );
+ f.write("visible.py", "value = 1\n");
+ f.commit();
+ expect(
+ f
+ .success("generate-in-scope-files", ["--scope", ".", "--diff-base", base])
+ .trim()
+ .split(/\r?\n/u),
+ ).toEqual(["visible.py"]);
+ expect(
+ f.rows("make-diff-rank-input", ["--base", base]).map((row) => row.path),
+ ).toEqual(["visible.py"]);
+});
+
+test("revision previews use the same read window as local files", () => {
+ const f = fixture();
+ f.write("example.py", "def first():\n pass\n");
+ const base = f.commit();
+ f.write(
+ "example.py",
+ "def first():\n pass\n#" +
+ "x".repeat(80_000) +
+ "\ndef outside_window():\n pass\n",
+ );
+ f.commit();
+ const local = f.localRows()[0]!.preview!;
+ const revision = f.rows("make-diff-rank-input", ["--base", base])[0]!
+ .preview!;
+ expect(revision).toBe(local);
+ expect(revision.startsWith("def first():\n pass")).toBe(true);
+ expect(Buffer.byteLength(revision)).toBeLessThanOrEqual(1024);
+ expect(revision).not.toContain("outside_window");
+});
+
+test("revision previews retain blobs with Git versions lacking NUL batch input", () => {
+ const f = fixture();
+ f.write("deleted.py", "deleted source\n");
+ f.write("changed.py", "old source\n");
+ const base = f.commit();
+ rmSync(join(f.repo, "deleted.py"));
+ f.write("changed.py", "new source\n");
+ f.write("empty.py", "");
+ f.write("binary", Buffer.from([0, 1, 2]));
+ f.commit();
+ const preload = join(f.root, "old-git.cjs");
+ const calls = join(f.root, "batch-arguments.jsonl");
+ writeFileSync(
+ preload,
+ `
+const cp = require("node:child_process");
+const spawn = cp.spawn;
+cp.spawn = (command, args, options) => {
+ if (args.includes("cat-file")) {
+ require("node:fs").appendFileSync(${JSON.stringify(calls)}, JSON.stringify(args) + "\\n");
+ args = args.map(arg => arg === "-z" || arg === "-Z" ? "--unsupported-batch-zero" : arg);
+ }
+ return spawn(command, args, options);
+};
+require("node:module").syncBuiltinESMExports();
+`,
+ );
+ const result = f.run(
+ "make-diff-rank-input",
+ ["--base", base, "--mode", "revisions"],
+ {
+ ...f.toolEnvironment,
+ NODE_OPTIONS: `${f.toolEnvironment["NODE_OPTIONS"] ?? ""} --require ${JSON.stringify(preload)}`,
+ },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ const rows = readFileSync(f.out, "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ expect(rows.find((row) => row.path === "deleted.py")?.preview).toBe("");
+ expect(rows.find((row) => row.path === "changed.py")?.preview).toBe(
+ "new source",
+ );
+ expect(rows.find((row) => row.path === "empty.py")?.preview).toBe("");
+ expect(rows.some((row) => row.path === "binary")).toBe(false);
+ const batches = readFileSync(calls, "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ expect(batches.length).toBeGreaterThan(0);
+ for (const args of batches) {
+ expect(args).not.toContain("-z");
+ expect(args).not.toContain("-Z");
+ }
+});
+
+for (const command of ["generate-in-scope-files", "make-repo-scope-input"]) {
+ for (const rawWindowsHome of [false, true]) {
+ test.skipIf(process.platform !== "win32")(
+ `${command} uses the selected project-local Codex ripgrep${rawWindowsHome ? " through the UTF-16 bridge" : ""}`,
+ () => {
+ const f = fixture();
+ rmSync(join(f.repo, ".git"), { recursive: true });
+ f.write(".ignore", "scope/ignored.py\n");
+ f.write("scope/visible.py");
+ f.write("scope/ignored.py");
+ const env = { ...f.toolEnvironment };
+ const pathKey =
+ Object.keys(env).find((key) => key.toUpperCase() === "PATH") ??
+ "PATH";
+ const ripgrep = Bun.which("rg", { PATH: env[pathKey] });
+ expect(ripgrep).not.toBeNull();
+ const runtime = join(
+ f.repo,
+ "node_modules",
+ "@openai",
+ `codex-win32-${process.arch}`,
+ "vendor",
+ "synthetic-target",
+ );
+ const codex = join(runtime, "bin", "codex.exe");
+ const tools = join(runtime, "codex-path");
+ mkdirSync(dirname(codex), { recursive: true });
+ mkdirSync(tools);
+ copyFileSync(node, codex);
+ copyFileSync(ripgrep!, join(tools, "rg.exe"));
+ copyFileSync(node, join(f.repo, "rg.exe"));
+ env[pathKey] = [f.repo, tools].join(delimiter);
+ env["CODEX_CLI_PATH"] = codex;
+ env["CODEX_SECURITY_GIT"] = "";
+ if (rawWindowsHome) env["HOME"] = `${f.root}${sep}home-\ud800`;
+ const result = f.run(
+ command,
+ command === "generate-in-scope-files"
+ ? ["--scope", "scope"]
+ : f.scopeArgs("scope"),
+ env,
+ );
+ expect(result.status, result.stderr).toBe(0);
+ const contents = readFileSync(f.out, "utf8");
+ expect(
+ command === "generate-in-scope-files"
+ ? contents.trim()
+ : JSON.parse(contents).path,
+ ).toBe("scope/visible.py");
+ },
+ );
+ }
+}
+
+for (const command of ["generate-in-scope-files", "make-repo-scope-input"]) {
+ for (const rawWindowsHome of [false, true]) {
+ test.skipIf(process.platform !== "win32")(
+ `${command} selects trusted ripgrep outside the repository${rawWindowsHome ? " through the UTF-16 bridge" : ""}`,
+ () => {
+ const f = fixture();
+ f.write("scope/visible.py");
+ const hostile = join(f.repo, "rg.exe");
+ copyFileSync(node, hostile);
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["scope"]');
+ const preload = join(f.root, "tool-launch.cjs");
+ const calls = join(f.root, "tool-launch.jsonl");
+ writeFileSync(
+ preload,
+ `
+const cp = require("node:child_process");
+const spawn = cp.spawn;
+const record = command => {
+ if (/(^|[\\\\/])rg(?:\\.exe|\\.com)?$/i.test(command))
+ require("node:fs").appendFileSync(${JSON.stringify(calls)}, JSON.stringify(command) + "\\n");
+};
+cp.spawn = (command, args, options) => {
+ record(command);
+ const child = spawn(command, args, options);
+ if (child.send) {
+ const send = child.send.bind(child);
+ child.send = (payload, ...rest) => {
+ if (payload.executable) record(Buffer.from(payload.executable, "base64").toString("utf16le"));
+ return send(payload, ...rest);
+ };
+ }
+ return child;
+};
+require("node:module").syncBuiltinESMExports();
+`,
+ );
+ const env = { ...f.toolEnvironment };
+ const pathKey =
+ Object.keys(env).find((key) => key.toUpperCase() === "PATH") ??
+ "PATH";
+ // The bridge case also checks Windows' implicit current-directory lookup.
+ if (!rawWindowsHome)
+ env[pathKey] = `${f.repo}${delimiter}${env[pathKey] ?? ""}`;
+ if (rawWindowsHome) {
+ env["HOME"] = `${f.root}${sep}home-\ud800`;
+ expect(env["HOME"].isWellFormed()).toBe(false);
+ }
+ env["CODEX_SECURITY_GIT"] = "";
+ env["NODE_OPTIONS"] =
+ `${env["NODE_OPTIONS"] ?? ""} --require ${JSON.stringify(preload)}`;
+ const result = f.run(
+ command,
+ command === "generate-in-scope-files"
+ ? ["--scope", "scope"]
+ : ["--scopes-file", scopes],
+ env,
+ );
+ expect(result.status, result.stderr).toBe(0);
+ const contents = readFileSync(f.out, "utf8");
+ expect(
+ command === "generate-in-scope-files"
+ ? contents.trim()
+ : JSON.parse(contents).path,
+ ).toBe("scope/visible.py");
+ const launched = readFileSync(calls, "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ expect(launched.length).toBeGreaterThan(0);
+ for (const executable of launched) {
+ expect(parse(executable).root).not.toBe("");
+ expect(executable).not.toBe(hostile);
+ }
+ },
+ );
+ }
+}
+
+for (const command of ["generate-in-scope-files", "make-repo-scope-input"]) {
+ for (const location of [
+ "relative",
+ "symlink parent",
+ "Windows bridge",
+ ] as const) {
+ test.skipIf(
+ location === "symlink parent"
+ ? process.platform === "win32"
+ : location === "Windows bridge" && process.platform !== "win32",
+ )(
+ `${command} resolves ${location} ripgrep PATH entries from the repository`,
+ () => {
+ const f = fixture();
+ f.write("scope/visible.py");
+ const env = { ...f.toolEnvironment };
+ const pathKey =
+ Object.keys(env).find((key) => key.toUpperCase() === "PATH") ??
+ "PATH";
+ const executable = Bun.which("rg", { PATH: env[pathKey] });
+ expect(executable).not.toBeNull();
+ const tools = join(
+ f.root,
+ ...(location === "symlink parent" ? ["physical"] : []),
+ "tools",
+ );
+ mkdirSync(tools, { recursive: true });
+ copyFileSync(
+ executable!,
+ join(tools, process.platform === "win32" ? "rg.exe" : "rg"),
+ );
+ if (location === "symlink parent") {
+ const child = join(f.root, "physical", "child");
+ mkdirSync(child);
+ symlinkSync(child, join(f.root, "link"), "dir");
+ }
+ env[pathKey] =
+ location === "symlink parent" ? "../link/../tools" : "../tools";
+ if (location === "Windows bridge") {
+ env["HOME"] = `${f.root}${sep}home-\ud800`;
+ expect(env["HOME"].isWellFormed()).toBe(false);
+ }
+ env["CODEX_SECURITY_GIT"] = "";
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["scope"]');
+ const result = f.run(
+ command,
+ command === "generate-in-scope-files"
+ ? ["--scope", "scope"]
+ : ["--scopes-file", scopes],
+ env,
+ );
+ expect(result.status, result.stderr).toBe(0);
+ const output = readFileSync(f.out, "utf8");
+ expect(
+ command === "generate-in-scope-files"
+ ? output.trim()
+ : JSON.parse(output).path,
+ ).toBe("scope/visible.py");
+ },
+ );
+ }
+}
+
+function runPathInventory(
+ f: ReturnType<typeof fixture>,
+ command: string,
+ env: NodeJS.ProcessEnv,
+ rawHome = false,
+) {
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["scope"]');
+ const args =
+ command === "generate-in-scope-files"
+ ? ["--scope", "scope"]
+ : ["--scopes-file", scopes];
+ if (!rawHome) return f.run(command, args, env);
+ return spawnSync(
+ "/bin/sh",
+ [
+ "-c",
+ 'HOME=$(printf "%s/\\377" "$1"); export HOME; shift; exec "$@"',
+ "inventory-raw-home",
+ f.root,
+ node,
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ command,
+ "--repo",
+ f.repo,
+ "--out",
+ f.out,
+ ...args,
+ ],
+ { env, encoding: "utf8" },
+ );
+}
+
+for (const command of ["generate-in-scope-files", "make-repo-scope-input"]) {
+ test.skipIf(process.platform === "win32")(
+ `${command} continues PATH lookup after a missing ripgrep interpreter`,
+ () => {
+ const f = fixture();
+ f.write("scope/visible.py");
+ const env = { ...f.toolEnvironment };
+ const first = join(f.root, "first");
+ mkdirSync(first);
+ writeFileSync(
+ join(first, "rg"),
+ `#!${join(f.root, "missing-interpreter")}\n`,
+ { mode: 0o700 },
+ );
+ env["PATH"] = `${first}${delimiter}${env["PATH"] ?? ""}`;
+ env["CODEX_SECURITY_GIT"] = "";
+ const result = runPathInventory(f, command, env);
+ expect(result.status, result.stderr).toBe(0);
+ const output = readFileSync(f.out, "utf8");
+ expect(
+ command === "generate-in-scope-files"
+ ? output.trim()
+ : JSON.parse(output).path,
+ ).toBe("scope/visible.py");
+ },
+ );
+ for (const lookup of [
+ "double-quoted semicolon",
+ "single-quoted semicolon",
+ "extension precedence",
+ ] as const) {
+ for (const bridge of lookup === "single-quoted semicolon"
+ ? [false]
+ : [false, true]) {
+ test.skipIf(process.platform !== "win32")(
+ `${command} retains Windows ${lookup} lookup${bridge ? " through the UTF-16 bridge" : ""}`,
+ () => {
+ const f = fixture();
+ f.write("scope/visible.py");
+ const env = { ...f.toolEnvironment };
+ const pathKey =
+ Object.keys(env).find((key) => key.toUpperCase() === "PATH") ??
+ "PATH";
+ const executable = Bun.which("rg", { PATH: env[pathKey] });
+ expect(executable).not.toBeNull();
+ const quoted = lookup !== "extension precedence";
+ const tools = join(f.root, quoted ? "tools;version" : "tools");
+ mkdirSync(tools);
+ // Ordinary Node lookup prefers COM; the existing raw bridge prefers EXE.
+ const extension = !quoted && !bridge ? "com" : "exe";
+ copyFileSync(executable!, join(tools, `rg.${extension}`));
+ if (!quoted)
+ copyFileSync(
+ node,
+ join(tools, `rg.${extension === "com" ? "exe" : "com"}`),
+ );
+ const quote = lookup === "single-quoted semicolon" ? "'" : '"';
+ env[pathKey] = quoted ? `${quote}${tools}${quote}` : tools;
+ if (bridge) {
+ env["HOME"] = `${f.root}${sep}home-\ud800`;
+ expect(env["HOME"].isWellFormed()).toBe(false);
+ }
+ env["CODEX_SECURITY_GIT"] = "";
+ const result = runPathInventory(f, command, env);
+ expect(result.status, result.stderr).toBe(0);
+ const output = readFileSync(f.out, "utf8");
+ expect(
+ command === "generate-in-scope-files"
+ ? output.trim()
+ : JSON.parse(output).path,
+ ).toBe("scope/visible.py");
+ },
+ );
+ }
+ }
+}
+
+for (const command of ["generate-in-scope-files", "make-repo-scope-input"])
+ for (const status of [126, 127])
+ for (const rawHome of [false, true])
+ for (const shebang of [false, true]) {
+ test.skipIf(process.platform === "win32")(
+ `${command} preserves a launched ripgrep wrapper's exit ${status}${rawHome ? " with raw HOME" : ""}${shebang ? "" : " without a shebang"}`,
+ () => {
+ const f = fixture();
+ rmSync(join(f.repo, ".git"), { recursive: true });
+ f.write("scope/visible.py");
+ const first = join(f.root, "first");
+ mkdirSync(first);
+ writeFileSync(
+ join(first, "rg"),
+ `${shebang ? "#!/bin/sh\n" : ""}printf "synthetic tool failure\\n\\00000000000-0000-4000-8000-000000000000\\000127\\000\\n" >&2\nexit ${status}\n`,
+ { mode: 0o700 },
+ );
+ writeFileSync(f.out, "previous\n");
+ const result = runPathInventory(
+ f,
+ command,
+ {
+ ...f.toolEnvironment,
+ PATH: `${first}${delimiter}${f.toolEnvironment["PATH"] ?? ""}`,
+ CODEX_SECURITY_GIT: "",
+ },
+ rawHome,
+ );
+ expect(result.status).toBe(
+ command === "generate-in-scope-files" ? 2 : 1,
+ );
+ expect(result.stderr).toContain("synthetic tool failure");
+ expect(result.stderr).toContain(
+ "\u000000000000-0000-4000-8000-000000000000\u0000127\u0000",
+ );
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ },
+ );
+ }
+
+for (const tool of ["git", "rg"])
+ for (const rawHome of [false, true])
+ test.skipIf(process.platform === "win32")(
+ `inventory does not pass its startup pipe to ${tool} wrappers${rawHome ? " with raw HOME" : ""}`,
+ () => {
+ const f = fixture();
+ f.write("scope/visible.py");
+ if (tool === "rg") rmSync(join(f.repo, ".git"), { recursive: true });
+ const executable = Bun.which(tool, { PATH: f.toolEnvironment["PATH"] });
+ expect(executable).not.toBeNull();
+ const tools = join(f.root, "tools");
+ mkdirSync(tools);
+ const wrapper = join(tools, tool);
+ const descriptor = join(f.root, "startup-pipe");
+ writeFileSync(
+ wrapper,
+ `#!/bin/sh
+state=closed
+for fd in /dev/fd/*; do
+ case "$fd" in /dev/fd/0|/dev/fd/1|/dev/fd/2) continue;; esac
+ if [ -p "$fd" ] || [ -S "$fd" ]; then state=inherited; fi
+done
+printf '%s' "$state" > '${descriptor.replaceAll("'", "'\\''")}'
+exec '${executable!.replaceAll("'", "'\\''")}' "$@"
+`,
+ { mode: 0o700 },
+ );
+ const result = runPathInventory(
+ f,
+ "make-repo-scope-input",
+ {
+ ...f.toolEnvironment,
+ PATH: `${tools}${delimiter}${f.toolEnvironment["PATH"] ?? ""}`,
+ CODEX_SECURITY_GIT: tool === "git" ? wrapper : "",
+ },
+ rawHome,
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(JSON.parse(readFileSync(f.out, "utf8")).path).toBe(
+ "scope/visible.py",
+ );
+ expect(readFileSync(descriptor, "utf8")).toBe("closed");
+ },
+ );
+
+for (const scope of [".", "src", "./src", "src/résumé.py"]) {
+ test(`path inventory preserves ripgrep spelling and byte order for ${scope}`, () => {
+ const f = fixture();
+ f.write("src/résumé.py");
+ f.write("src/alpha.py");
+ f.write("src/binary", Buffer.from([0]));
+ f.write(".hidden/source.py");
+ f.write(".gitignore", "ignored/\n");
+ f.write("ignored/untracked.py");
+ f.write("ignored/tracked.py");
+ git(f.repo, "add", "--force", "ignored/tracked.py");
+ const args = [
+ "--files",
+ "--null",
+ "--hidden",
+ "--path-separator",
+ "/",
+ "--glob",
+ "!**/.git",
+ "--glob",
+ "!**/.git/**",
+ "--",
+ scope,
+ ];
+ const rg = spawnSync("rg", args, { cwd: f.repo, env: f.toolEnvironment });
+ expect(rg.status, rg.error?.message ?? rg.stderr?.toString()).toBe(0);
+ const expected = rg.stdout.toString().split("\0").filter(Boolean);
+ if (scope === ".") expected.push("./ignored/tracked.py");
+ expect(f.success("generate-in-scope-files", ["--scope", scope])).toBe(
+ expected
+ .map((path) => Buffer.from(path + "\n"))
+ .sort(Buffer.compare)
+ .map(String)
+ .join(""),
+ );
+ expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual(
+ [],
+ );
+ });
+}
+
+test("make-repo-scope-input retains explicit and ignored tracked files without widening directory scope", () => {
+ const f = fixture();
+ f.write("src/source.py");
+ f.write("src/entrypoint", "run\n");
+ f.write("outside.py");
+ f.write(".gitignore", "src/ignored*\n");
+ f.write("src/ignored-tracked.py");
+ f.write("src/ignored-untracked.py");
+ f.write("src/binary", Buffer.from([0]));
+ f.write("direct.bin", Buffer.from([0]));
+ git(f.repo, "add", "--force", "src/ignored-tracked.py");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify(["src", "direct.bin", "src/source.py"]));
+ const rows = f.rows("make-repo-scope-input", ["--scopes-file", scopes]);
+ expect(rows.map((row) => row.path)).toEqual([
+ "direct.bin",
+ "src/binary",
+ "src/entrypoint",
+ "src/ignored-tracked.py",
+ "src/source.py",
+ ]);
+});
+
+for (const encoding of ["utf8", "utf16le", "utf16be"] as const) {
+ for (const mode of ["revisions", "local-patch"]) {
+ test(`${mode} previews decode ${encoding} and detect binary bytes beyond the prefix`, () => {
+ const f = fixture();
+ f.write("existing.py");
+ const base = f.commit();
+ const source = " value = 'café 😀 literal' \n\n return value \n";
+ const bytes =
+ encoding === "utf8"
+ ? Buffer.from(source)
+ : Buffer.concat([
+ Buffer.from([0xff, 0xfe]),
+ Buffer.from(source, "utf16le"),
+ ]);
+ if (encoding === "utf16be") bytes.swap16();
+ f.write("text", bytes);
+ const lateBinary =
+ encoding === "utf8"
+ ? Buffer.concat([Buffer.alloc(96 * 1024, 120), Buffer.from([0])])
+ : Buffer.concat([
+ Buffer.from([0xff, 0xfe]),
+ Buffer.from("x".repeat(96 * 1024) + "\0", "utf16le"),
+ ]);
+ if (encoding === "utf16be") lateBinary.swap16();
+ f.write("encoded-binary", lateBinary);
+ f.write(
+ "late-binary",
+ Buffer.concat([Buffer.alloc(96 * 1024, 120), Buffer.from([0])]),
+ );
+ for (const bom of [
+ [0xff, 0xfe],
+ [0xfe, 0xff],
+ ])
+ f.write(
+ `late-bom-${bom[0]}`,
+ Buffer.concat([
+ Buffer.alloc(64 * 1024, 120),
+ Buffer.from([...bom, 104, 0, 105, 0]),
+ ]),
+ );
+ if (mode === "revisions") f.commit();
+ const rows = f.rows("make-diff-rank-input", [
+ "--base",
+ base,
+ "--mode",
+ mode,
+ ]);
+ expect(rows.find((row) => row.path === "text")?.preview).toBe(
+ source.slice(0, -1),
+ );
+ expect(
+ rows.some(
+ (row) =>
+ row.path === "late-binary" ||
+ row.path === "encoded-binary" ||
+ row.path.startsWith("late-bom-"),
+ ),
+ ).toBe(false);
+ });
+ }
+}
+
+for (const budget of [0, -1, 2, 30, 220, 1024]) {
+ test(`preview budget ${budget} preserves source and complete UTF-8 units`, () => {
+ const f = fixture(),
+ source = Array.from(
+ { length: 40 },
+ (_, index) =>
+ `line_${String(index).padStart(2, "0")} ${"😀".repeat(20)}`,
+ ).join("\n");
+ f.write("source", source);
+ const preview = f.localRows(["--preview-bytes", String(budget)])[0]!
+ .preview!;
+ expect(Buffer.byteLength(preview)).toBeLessThanOrEqual(Math.max(0, budget));
+ expect(preview).not.toContain("\ufffd");
+ if (budget <= 0) expect(preview).toBe("");
+ else if (budget <= 30) expect(source.startsWith(preview)).toBe(true);
+ else {
+ expect(preview).toContain("line_39");
+ expect(preview).toContain("...");
+ }
+ });
+}
+
+test("small previews retain complete bodies, literal replacement characters, and line whitespace", () => {
+ const f = fixture();
+ f.write(
+ "source",
+ "\n \t\n text = 'two spaces\tand\u0085a separator �' \r\n return text \r\n\n",
+ );
+ expect(f.localRows()[0]!.preview).toBe(
+ " text = 'two spaces\tand\u0085a separator �' \n return text ",
+ );
+});
+
+test("explicit scope names are literal and JSONL escapes Unicode separators", () => {
+ const f = fixture(),
+ name = "audit\u0085line\u2028paragraph\u2029.py";
+ f.write(name);
+ f.write("~literal/file");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify([name, "~literal"]));
+ const text = f.success("make-repo-scope-input", ["--scopes-file", scopes]);
+ expect(text).not.toMatch(/[\u0085\u2028\u2029]/u);
+ expect(
+ text
+ .split(/\r?\n/u)
+ .filter(Boolean)
+ .map((line) => JSON.parse(line).path),
+ ).toEqual([name, "~literal/file"]);
+});
+
+for (const mode of ["revisions", "local-patch"]) {
+ test(`${mode} selects added, changed, deleted, and renamed regular files`, () => {
+ const f = fixture();
+ f.write("changed.py", "old\n");
+ f.write("deleted.py", "delete\n");
+ f.write("renamed.py", "rename\n");
+ const base = f.commit();
+ f.write("changed.py", "new\n");
+ rmSync(join(f.repo, "deleted.py"));
+ git(f.repo, "mv", "renamed.py", "moved.py");
+ f.write("new-file", "added\n");
+ git(f.repo, "add", ".");
+ f.write("unstaged.py", "unstaged\n");
+ if (mode === "revisions") {
+ f.commit();
+ f.write("changed.py", "worktree must not leak\n");
+ }
+ const args = ["--base", base, "--mode", mode];
+ const rows = f.rows("make-diff-rank-input", args);
+ expect(rows.find((row) => row.path === "changed.py")?.preview).toBe("new");
+ expect(rows.find((row) => row.path === "deleted.py")?.preview).toBe("");
+ expect(rows.map((row) => row.path)).toEqual([
+ "changed.py",
+ "deleted.py",
+ "moved.py",
+ "new-file",
+ "unstaged.py",
+ ]);
+ expect(
+ f.success("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ mode,
+ ]),
+ ).toBe(rows.map((row) => row.path + "\n").join(""));
+ });
+}
+
+for (const [mode, undoStagedChange] of [
+ ["revisions", false],
+ ["local-patch", false],
+ ["local-patch", true],
+] as const) {
+ test(`${mode} inventories type changes with staged change undone ${undoStagedChange}`, () => {
+ const f = fixture();
+ f.write("routes.py");
+ const changed = join(f.repo, "changed.py");
+ symlinkSync("routes.py", changed, "file");
+ const base = f.commit();
+ rmSync(changed);
+ f.write("changed.py", "changed = True\n");
+ if (undoStagedChange) {
+ git(f.repo, "add", ".");
+ rmSync(changed);
+ symlinkSync("routes.py", changed, "file");
+ } else if (mode === "revisions") f.commit();
+ const inventory = f
+ .success("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ mode,
+ ])
+ .split("\n");
+ const ranked = f.rows("make-diff-rank-input", [
+ "--base",
+ base,
+ "--mode",
+ mode,
+ ]);
+ expect(inventory.includes("changed.py")).toBe(!undoStagedChange);
+ expect(ranked.some(({ path }) => path === "changed.py")).toBe(
+ !undoStagedChange,
+ );
+ });
+}
+
+for (const mode of ["revisions", "local-patch"] as const) {
+ for (const replacement of ["symlink", "gitlink"] as const) {
+ test(`${mode} inventories exclude files replaced by ${replacement}`, () => {
+ const f = fixture();
+ f.write("README.md", "fixture\n");
+ const source = f.write("replaced.py", "old source\n");
+ const base = f.commit();
+ rmSync(source);
+ if (replacement === "symlink") {
+ symlinkSync("README.md", source, "file");
+ git(f.repo, "add", "--", "replaced.py");
+ } else {
+ mkdirSync(source);
+ git(
+ f.repo,
+ "update-index",
+ "--add",
+ "--cacheinfo",
+ "160000," + base + ",replaced.py",
+ );
+ f.write(
+ ".gitmodules",
+ '[submodule "fixture"]\n\tpath = replaced.py\n\turl = ../fixture\n',
+ );
+ git(f.repo, "add", ".gitmodules");
+ }
+ f.write("visible.py");
+ git(f.repo, "add", "visible.py");
+ if (mode === "revisions")
+ git(f.repo, "commit", "-qm", "Replace source type");
+ const expected =
+ replacement === "gitlink"
+ ? [".gitmodules", "visible.py"]
+ : ["visible.py"];
+ expect(
+ f
+ .success("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ mode,
+ ])
+ .trim()
+ .split("\n"),
+ ).toEqual(expected);
+ expect(
+ f
+ .rows("make-diff-rank-input", ["--base", base, "--mode", mode])
+ .map(({ path }) => path),
+ ).toEqual(expected);
+ });
+ }
+}
+
+for (const location of ["outside", "repository-extra", "inside"]) {
+ for (const binary of [false, true]) {
+ test(`local diff inventory confines ${location} directory links with ${binary ? "binary" : "text"} contents`, () => {
+ const f = fixture();
+ const contents = binary ? Buffer.from("\0selected\n") : "selected\n";
+ f.write("inside/source.py", contents);
+ f.write("safe.py", "before\n");
+ const base = f.commit();
+ f.write("src/nested/source.py", "staged\n");
+ git(f.repo, "add", "src/nested/source.py");
+ f.write("safe.py", "after\n");
+ const destination =
+ location === "inside" ? join(f.repo, "inside") : join(f.root, location);
+ if (location !== "inside") {
+ mkdirSync(destination);
+ writeFileSync(join(destination, "source.py"), contents);
+ }
+ rmSync(join(f.repo, "src", "nested"), { recursive: true });
+ symlinkSync(destination, join(f.repo, "src", "nested"), "junction");
+
+ const retained = location === "inside" && !binary;
+ expect(
+ f.success("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ "local-patch",
+ ]),
+ ).toBe(`safe.py\n${retained ? "src/nested/source.py\n" : ""}`);
+ expect(
+ f.rows("make-diff-rank-input", [
+ "--base",
+ base,
+ "--mode",
+ "local-patch",
+ ]),
+ ).toEqual([
+ { path: "safe.py", area: "diff", preview: "after" },
+ ...(location !== "inside" || !binary
+ ? [
+ {
+ path: "src/nested/source.py",
+ area: "diff",
+ preview: retained ? "selected" : "",
+ },
+ ]
+ : []),
+ ]);
+ });
+ }
+}
+
+test.skipIf(process.platform === "win32")(
+ "local diff inventory does not open outside directory-link targets",
+ () => {
+ const f = fixture();
+ f.write("safe.py", "before\n");
+ const base = f.commit();
+ f.write("nested/source.py", "staged\n");
+ git(f.repo, "add", "nested/source.py");
+ f.write("safe.py", "after\n");
+ const outside = join(f.root, "outside");
+ mkdirSync(outside);
+ const outsideFile = join(outside, "source.py");
+ writeFileSync(outsideFile, "synthetic outside contents\n");
+ rmSync(join(f.repo, "nested"), { recursive: true });
+ symlinkSync(outside, join(f.repo, "nested"), "dir");
+ const marker = join(f.root, "read-attempt");
+ writeFileSync(marker, "untouched");
+ const preload = join(f.root, "observe-open.cjs");
+ writeFileSync(
+ preload,
+ `
+const fs = require("node:fs");
+const open = fs.openSync;
+fs.openSync = (path, ...args) => {
+ if (${JSON.stringify([join(f.repo, "nested", "source.py"), outsideFile])}.includes(String(path))) {
+ fs.writeFileSync(${JSON.stringify(marker)}, "opened");
+ throw new Error("Synthetic outside-file read tripwire");
+ }
+ return open(path, ...args);
+};
+require("node:module").syncBuiltinESMExports();
+`,
+ );
+ const result = f.run(
+ "generate-in-scope-files",
+ ["--scope", ".", "--diff-base", base, "--diff-mode", "local-patch"],
+ {
+ ...f.toolEnvironment,
+ NODE_OPTIONS: `${f.toolEnvironment["NODE_OPTIONS"] ?? ""} --require ${JSON.stringify(preload)}`,
+ },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(f.out, "utf8")).toBe("safe.py\n");
+ expect(readFileSync(marker, "utf8")).toBe("untouched");
+ },
+);
+
+test("local diff inventory skips unreadable files", () => {
+ const f = fixture();
+ f.write("tracked.py");
+ const base = f.commit();
+ f.write("unreadable.py");
+ f.write("readable.py");
+ const preload = join(f.root, "read-error.cjs");
+ writeFileSync(
+ preload,
+ `
+const failure = () => { throw Object.assign(new Error("Synthetic unreadable file"), { code: "EACCES" }); };
+const fs = require("node:fs");
+const open = fs.openSync;
+fs.openSync = (path, ...args) => String(path).endsWith("unreadable.py") ? failure() : open(path, ...args);
+require("node:module").syncBuiltinESMExports();
+const Module = require("node:module");
+const native = Module._extensions[".node"];
+Module._extensions[".node"] = (loaded, path) => {
+ native(loaded, path);
+ if (!path.endsWith("windows.node")) return;
+ loaded.exports = new Proxy(loaded.exports, { get(exports, key) {
+ const value = Reflect.get(exports, key);
+ if (key !== "openWindowsFile") return value;
+ return (path, ...args) => {
+ const result = value(path, ...args);
+ if (path.toString("utf16le").endsWith("unreadable.py") && result.handle) {
+ return { ...result, handle: new Proxy(result.handle, { get(handle, key) {
+ if (key === "read") return failure;
+ const member = Reflect.get(handle, key);
+ return typeof member === "function" ? member.bind(handle) : member;
+ }}) };
+ }
+ return result;
+ };
+ }});
+};
+`,
+ );
+ const result = f.run(
+ "generate-in-scope-files",
+ ["--scope", ".", "--diff-base", base, "--diff-mode", "local-patch"],
+ {
+ ...f.toolEnvironment,
+ NODE_OPTIONS: `${f.toolEnvironment["NODE_OPTIONS"] ?? ""} --require ${JSON.stringify(preload)}`,
+ },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(f.out, "utf8")).toBe("readable.py\n");
+});
+
+test.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ "ignored tracked access errors preserve the existing inventory",
+ () => {
+ const f = fixture();
+ f.write(".gitignore", "ignored/\n");
+ f.write("ignored/tracked.py");
+ f.write("visible.py");
+ git(f.repo, "add", "--force", "ignored/tracked.py");
+ writeFileSync(f.out, "previous\n");
+ const directory = join(f.repo, "ignored");
+ chmodSync(directory, 0);
+ try {
+ const result = f.run("generate-in-scope-files", ["--scope", "."]);
+ expect(result.status).toBe(2);
+ expect(result.stderr).toMatch(/EACCES|permission denied/iu);
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ } finally {
+ chmodSync(directory, 0o700);
+ }
+ },
+);
+
+test.skipIf(process.platform === "win32")(
+ "make-repo-scope-input skips candidates that become inaccessible after enumeration",
+ () => {
+ const f = fixture();
+ const blocked = f.write("blocked.py");
+ f.write("visible.py");
+ git(f.repo, "add", ".");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["."]');
+ const preload = join(f.root, "stat-error.cjs");
+ writeFileSync(
+ preload,
+ `const fs = require("node:fs");
+const original = fs.statSync;
+fs.statSync = (path, ...args) => {
+ if (String(path) === ${JSON.stringify(blocked)})
+ throw Object.assign(new Error("Synthetic access denial"), {code: "EACCES"});
+ return original(path, ...args);
+};
+require("node:module").syncBuiltinESMExports();
+`,
+ );
+ const result = f.run("make-repo-scope-input", ["--scopes-file", scopes], {
+ ...f.toolEnvironment,
+ NODE_OPTIONS: `${f.toolEnvironment["NODE_OPTIONS"] ?? ""} --require ${JSON.stringify(preload)}`,
+ });
+ expect(result.status, result.stderr).toBe(0);
+ expect(
+ readFileSync(f.out, "utf8")
+ .trim()
+ .split("\n")
+ .map((row) => JSON.parse(row).path),
+ ).toEqual(["visible.py"]);
+ },
+);
+
+test("failed inventory generation preserves the existing file", () => {
+ const f = fixture();
+ f.write("source");
+ writeFileSync(f.out, "previous\n");
+ for (const args of [
+ ["--scope", "missing"],
+ ["--scope", ".."],
+ ["--scope", ".", "--diff-base", "missing-revision"],
+ ]) {
+ expect(f.run("generate-in-scope-files", args).status).toBe(2);
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ }
+ const absent = f.run("generate-in-scope-files", ["--scope", "."], {
+ ...process.env,
+ PATH: f.root,
+ });
+ expect(absent.status).toBe(2);
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+});
+
+test("make-repo-scope-input ignores tracked descendants replaced by symlinks", () => {
+ const f = fixture();
+ f.write("src/nested/source");
+ f.commit();
+ rmSync(join(f.repo, "src", "nested"), { recursive: true });
+ const outside = join(f.root, "outside");
+ mkdirSync(outside);
+ writeFileSync(join(outside, "source"), "external\n");
+ symlinkSync(outside, join(f.repo, "src", "nested"), "junction");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify(["src"]));
+ expect(f.rows("make-repo-scope-input", ["--scopes-file", scopes])).toEqual(
+ [],
+ );
+ writeFileSync(scopes, JSON.stringify(["src/nested/../source"]));
+ expect(
+ f.run("make-repo-scope-input", ["--scopes-file", scopes]).status,
+ ).not.toBe(0);
+});
+
+test.skipIf(process.platform === "win32")(
+ "line-breaking paths fail without replacing inventory",
+ () => {
+ for (const [name, scope, diff] of [
+ ["ignored/bad\rname", ".", false],
+ ["app/bad\nname", "./app", false],
+ ["app/bad\r\nname", "app/bad\r\nname", false],
+ ["app/bad\nname", ".", true],
+ ] as const) {
+ const f = fixture();
+ f.write(".gitignore", "ignored/\n");
+ const base = f.commit();
+ f.write(name);
+ if (name.startsWith("ignored/"))
+ git(f.repo, "add", "--force", "--", name);
+ writeFileSync(f.out, "previous\n");
+ const result = f.run("generate-in-scope-files", [
+ "--scope",
+ scope,
+ ...(diff ? ["--diff-base", base, "--diff-mode", "local-patch"] : []),
+ ]);
+ expect(result.status, result.stderr).toBe(2);
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ }
+ },
+);
+
+for (const mode of ["revisions", "local-patch"]) {
+ test.skipIf(process.platform !== "linux")(
+ `${mode} diff inventory preserves raw POSIX filename bytes`,
+ () => {
+ const f = fixture();
+ f.write("source");
+ const base = f.commit();
+ const name = Buffer.from([0x63, 0x61, 0x66, 0xe9, 0x2e, 0x70, 0x79]);
+ writeFileSync(Buffer.concat([Buffer.from(f.repo + "/"), name]), "text\n");
+ if (mode === "revisions") f.commit();
+ const result = f.run("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ mode,
+ ]);
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(f.out)).toEqual(
+ Buffer.concat([name, Buffer.from("\n")]),
+ );
+ },
+ );
+}
+
+test("inventory excludes nested Git metadata and handles long destination names", () => {
+ const f = fixture();
+ f.write("nested/source");
+ git(join(f.repo, "nested"), "init", "-q");
+ for (const name of ["a".repeat(251) + ".txt", "文".repeat(83) + ".txt"]) {
+ const out = join(f.root, name);
+ const result = f.run("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--out",
+ out,
+ ]);
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(out, "utf8")).toBe("./nested/source\n");
+ }
+});
+
+test("scope inventory rejects symbolic-link components before lexical parent traversal", () => {
+ const f = fixture();
+ f.write("src/source");
+ symlinkSync(join(f.repo, "src"), join(f.repo, "alias"), "junction");
+ const scopes = join(f.root, "scopes.json");
+ for (const scope of [
+ "alias/source",
+ "alias/../src/source",
+ `${f.root}${sep}.${sep}repository${sep}alias${sep}..${sep}src${sep}source`,
+ ]) {
+ writeFileSync(scopes, JSON.stringify([scope]));
+ const result = f.run("make-repo-scope-input", ["--scopes-file", scopes]);
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain("symbolic links");
+ }
+});
+
+test.skipIf(process.platform === "win32")(
+ "path inventory refuses output symlinks and creates private replacement files",
+ () => {
+ const f = fixture();
+ f.write("source");
+ const target = join(f.root, "existing");
+ writeFileSync(target, "untouched\n");
+ symlinkSync(target, f.out);
+ const rejected = f.run("generate-in-scope-files", ["--scope", "."]);
+ expect(rejected.status).toBe(2);
+ expect(rejected.stderr).toContain("symbolic link");
+ expect(readFileSync(target, "utf8")).toBe("untouched\n");
+ rmSync(f.out);
+ f.success("generate-in-scope-files", ["--scope", "."]);
+ expect(statSync(f.out).mode & 0o777).toBe(0o600);
+ },
+);
+
+for (const ending of ["\n", "\r"])
+ test.skipIf(process.platform === "win32")(
+ `launchers preserve POSIX roots ending ${JSON.stringify(ending)} and scoped names through Git and ripgrep`,
+ () => {
+ const f = fixture();
+ f.write("source", "selected revision\n");
+ const base = f.commit();
+ f.write("source", "current revision\n");
+ f.commit();
+ const rawRoot = Buffer.concat([
+ Buffer.from(f.root + "/raw-"),
+ pathNameBytes,
+ Buffer.from("'`$" + ending),
+ ]);
+ renameSync(f.repo, rawRoot);
+ const rawScope = Buffer.concat([
+ Buffer.from("scope-"),
+ pathNameBytes,
+ Buffer.from("'`$"),
+ ]);
+ mkdirSync(Buffer.concat([rawRoot, Buffer.from("/"), rawScope]));
+ writeFileSync(
+ Buffer.concat([
+ rawRoot,
+ Buffer.from("/"),
+ rawScope,
+ Buffer.from("/entry"),
+ ]),
+ "scoped source\n",
+ );
+ const literal = (bytes: Buffer) =>
+ [...bytes]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ const launcher = join(
+ PLUGIN_ROOT,
+ "scripts",
+ "launch_codex_security_mcp",
+ );
+ const run = (command: string, scoped: boolean) =>
+ spawnSync(
+ "/bin/sh",
+ [
+ "-c",
+ [
+ `repository=$(printf '%b.' '${literal(rawRoot)}'); repository=\${repository%.}`,
+ `scope=$(printf '%b.' '${literal(rawScope)}'); scope=\${scope%.}`,
+ `exec "$1" --helper "$2" --repo "$repository" --out "$3" ${scoped ? '--scope "$scope"' : '--base "$4" --head HEAD'}`,
+ ].join("\n"),
+ "inventory-fixture",
+ launcher,
+ command,
+ f.out,
+ base,
+ ],
+ { encoding: "utf8", env: f.toolEnvironment },
+ );
+ const diff = run("make-diff-rank-input", false);
+ expect(diff.status, diff.stderr).toBe(0);
+ expect(JSON.parse(readFileSync(f.out, "utf8")).preview).toBe(
+ "current revision",
+ );
+ const inventory = run("generate-in-scope-files", true);
+ expect(inventory.status, inventory.stderr).toBe(0);
+ expect(readFileSync(f.out)).toEqual(
+ Buffer.concat([rawScope, Buffer.from("/entry\n")]),
+ );
+ },
+ );
+
+test("Git inventory clears inherited repository selection and keeps forced tracked files", () => {
+ const f = fixture();
+ f.write(".gitignore", "tracked.py\n");
+ f.write("tracked.py");
+ git(f.repo, "add", "--force", "tracked.py");
+ const env = {
+ ...process.env,
+ [process.platform === "win32" ? "gIt_DiR" : "GIT_DIR"]: join(
+ f.root,
+ "missing-git",
+ ),
+ [process.platform === "win32" ? "gIt_WoRk_TrEe" : "GIT_WORK_TREE"]: f.root,
+ };
+ const result = f.run("make-repo-scope-input", f.scopeArgs(), env);
+ expect(result.status, result.stderr).toBe(0);
+ expect(
+ readFileSync(f.out, "utf8")
+ .trim()
+ .split(/\r?\n/u)
+ .map((line) => JSON.parse(line).path),
+ ).toEqual([".gitignore", "tracked.py"]);
+});
+
+test.skipIf(process.platform === "win32")(
+ "path inventories exceed the default execFile output buffer",
+ () => {
+ const f = fixture(),
+ bin = join(f.root, "bin");
+ mkdirSync(bin);
+ const command = join(bin, "rg"),
+ count = 20_000,
+ prefix = "source-".repeat(15);
+ writeFileSync(
+ command,
+ `#!${node}\nfor(let index=0;index<${count};index++) process.stdout.write(${JSON.stringify(prefix)}+String(index).padStart(5,'0')+'\\0');\n`,
+ { mode: 0o700 },
+ );
+ const result = f.run("generate-in-scope-files", ["--scope", "."], {
+ ...process.env,
+ PATH: bin,
+ CODEX_SECURITY_GIT: "",
+ });
+ expect(result.status, result.stderr).toBe(0);
+ const rows = readFileSync(f.out, "utf8").trimEnd().split("\n");
+ expect(rows.length).toBe(count);
+ expect(rows[0]).toBe(prefix + "00000");
+ expect(rows.at(-1)).toBe(prefix + "19999");
+ },
+);
+
+test("explicit scopes preserve literal glob, tilde and colon filenames", () => {
+ const f = fixture();
+ const files =
+ process.platform === "win32"
+ ? ["src/[slug]/page.tsx", "~/example.ts"]
+ : [
+ "src/[slug]/page.tsx",
+ "src/star*file.ts",
+ "src/question?file.ts",
+ "~/example.ts",
+ "module:handler.ts",
+ ];
+ for (const path of [
+ ...files,
+ "src/s/page.tsx",
+ "src/l/page.tsx",
+ "src/starOtherfile.ts",
+ "src/questionXfile.ts",
+ ])
+ f.write(path);
+ f.commit();
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify(["src/[slug]", ...files.slice(1)]));
+ expect(
+ f
+ .rows("make-repo-scope-input", ["--scopes-file", scopes])
+ .map((row) => row.path),
+ ).toEqual([...files].sort());
+});
+
+for (const rawExecutable of [false, true])
+ test.skipIf(process.platform === "win32")(
+ `Git children retain raw HOME/PATH and explicit Git filters (raw executable: ${rawExecutable})`,
+ () => {
+ const f = fixture();
+ f.write("visible.py");
+ f.write("hidden.py");
+ const home = Buffer.concat([
+ Buffer.from(f.root + "/home-"),
+ pathNameBytes,
+ ]);
+ const bin = Buffer.concat([Buffer.from(f.root + "/bin-"), pathNameBytes]);
+ mkdirSync(home);
+ mkdirSync(bin);
+ writeFileSync(
+ Buffer.concat([home, Buffer.from("/.gitconfig")]),
+ "[core]\nexcludesFile = ~/.gitignore_global\n",
+ );
+ writeFileSync(
+ Buffer.concat([home, Buffer.from("/.gitignore_global")]),
+ "hidden.py\n",
+ );
+ writeFileSync(
+ Buffer.concat([bin, Buffer.from("/inventory-child-tool")]),
+ '#!/bin/sh\nprintf executed > "$INVENTORY_CHILD_MARKER"\n',
+ { mode: 0o700 },
+ );
+ const gitPath = Buffer.concat([
+ Buffer.from(f.root + "/git-"),
+ rawExecutable ? pathNameBytes : Buffer.from("wrapper"),
+ ]);
+ const hostGit = Bun.which("git")!;
+ const quote = (value: string) =>
+ "'" + value.replaceAll("'", "'\\''") + "'";
+ writeFileSync(
+ gitPath,
+ `#!/bin/sh\nset -e\ntest "$GIT_LITERAL_PATHSPECS" = 1\ntest -z "\${GIT_DIR+x}"\ninventory-child-tool\nexec ${quote(hostGit)} "$@"\n`,
+ { mode: 0o700 },
+ );
+ const octal = (bytes: Buffer) =>
+ [...bytes]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ const marker = join(f.root, "child-ran");
+ const script = [
+ `HOME=$(printf '%b' '${octal(home)}'); export HOME`,
+ `raw_bin=$(printf '%b' '${octal(bin)}'); PATH="$raw_bin"; export PATH`,
+ `CODEX_SECURITY_GIT=$(printf '%b' '${octal(gitPath)}'); export CODEX_SECURITY_GIT`,
+ 'exec "$1" "$2" make-repo-scope-input --repo "$3" --out "$4" --scopes-file "$5"',
+ ].join("\n");
+ const result = spawnSync(
+ "/bin/sh",
+ [
+ "-c",
+ script,
+ "inventory-env",
+ node,
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ f.repo,
+ f.out,
+ f.scopeArgs()[1]!,
+ ],
+ {
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ GIT_CONFIG_GLOBAL: undefined,
+ GIT_CONFIG_NOSYSTEM: "1",
+ GIT_DIR: join(f.root, "wrong-git"),
+ GIT_LITERAL_PATHSPECS: "0",
+ INVENTORY_CHILD_MARKER: marker,
+ },
+ },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(
+ readFileSync(f.out, "utf8")
+ .trim()
+ .split(/\r?\n/u)
+ .map((line) => JSON.parse(line).path),
+ ).toEqual(["visible.py"]);
+ expect(readFileSync(marker, "utf8")).toBe("executed");
+ },
+ );
+
+test.skipIf(process.platform === "win32")(
+ "ripgrep children retain raw HOME and PATH",
+ () => {
+ const f = fixture();
+ f.write("visible.py");
+ const home = Buffer.concat([Buffer.from(f.root + "/home-"), pathNameBytes]);
+ const bin = Buffer.concat([Buffer.from(f.root + "/bin-"), pathNameBytes]);
+ mkdirSync(home);
+ mkdirSync(bin);
+ writeFileSync(Buffer.concat([home, Buffer.from("/marker")]), "raw-home\n");
+ writeFileSync(
+ Buffer.concat([bin, Buffer.from("/rg")]),
+ '#!/bin/sh\nset -e\nIFS= read -r value < "$HOME/marker"\ntest "$value" = raw-home\nprintf "./visible.py\\0"\n',
+ { mode: 0o700 },
+ );
+ const octal = (bytes: Buffer) =>
+ [...bytes]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ const script = [
+ `HOME=$(printf '%b' '${octal(home)}'); export HOME`,
+ `PATH=$(printf '%b' '${octal(bin)}'); export PATH`,
+ 'exec "$1" "$2" generate-in-scope-files --repo "$3" --scope . --out "$4"',
+ ].join("\n");
+ const result = spawnSync(
+ "/bin/sh",
+ [
+ "-c",
+ script,
+ "inventory-env",
+ node,
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ f.repo,
+ f.out,
+ ],
+ { encoding: "utf8", env: { ...process.env, CODEX_SECURITY_GIT: "" } },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(f.out, "utf8")).toBe("./visible.py\n");
+ },
+);
+
+for (const tool of ["git", "rg"])
+ for (const rawLoader of [false, true])
+ test.skipIf(process.platform === "win32")(
+ `${tool} children retain loader configuration with ${rawLoader ? "raw loader values" : "raw HOME"}`,
+ () => {
+ const f = fixture();
+ f.write("visible.py");
+ const tools = join(f.root, "tools");
+ mkdirSync(tools);
+ const value = Buffer.concat([
+ Buffer.from(f.root + "/libraries-"),
+ rawLoader ? Buffer.from([0xff, 0xfe]) : Buffer.from("ascii"),
+ Buffer.from("' $value\n"),
+ ]);
+ const octal = [...value]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ const quote = (text: string) =>
+ "'" + text.replaceAll("'", "'\\''") + "'";
+ const marker = join(f.root, "child-ran");
+ writeFileSync(
+ join(tools, tool),
+ [
+ "#!/bin/sh",
+ "set -e",
+ `expected=$(printf '%b.' '${octal}'); expected=\${expected%.}`,
+ 'test "$LD_LIBRARY_PATH" = "$expected"',
+ 'test "$LD_PROFILE_OUTPUT" = "$expected"',
+ 'test "${LD_PROFILE+x}" = x && test -z "$LD_PROFILE"',
+ 'test -z "${LD_PRELOAD+x}"',
+ 'test "$INVENTORY_CALLER_VALUE" = preserved',
+ ...(tool === "git"
+ ? ['test -z "${GIT_DIR+x}"', 'test "$GIT_LITERAL_PATHSPECS" = 1']
+ : []),
+ `printf executed > ${quote(marker)}`,
+ tool === "git"
+ ? `exec ${quote(Bun.which("git")!)} "$@"`
+ : "printf './visible.py\\0'",
+ ].join("\n") + "\n",
+ { mode: 0o700 },
+ );
+ const result = spawnSync(
+ "/bin/sh",
+ [
+ "-c",
+ [
+ `value=$(printf '%b.' '${octal}'); value=\${value%.}`,
+ 'LD_LIBRARY_PATH="$value"; export LD_LIBRARY_PATH',
+ 'LD_PROFILE_OUTPUT="$value"; export LD_PROFILE_OUTPUT',
+ ...(rawLoader ? [] : ["HOME=$(printf '\\377'); export HOME"]),
+ 'exec "$@"',
+ ].join("\n"),
+ "inventory-loader-environment",
+ node,
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "generate-in-scope-files",
+ "--repo",
+ f.repo,
+ "--scope",
+ ".",
+ "--out",
+ f.out,
+ ],
+ {
+ encoding: "utf8",
+ env: {
+ ...f.toolEnvironment,
+ PATH: `${tools}${delimiter}${f.toolEnvironment["PATH"] ?? ""}`,
+ CODEX_SECURITY_GIT: tool === "git" ? join(tools, tool) : "",
+ GIT_DIR: join(f.root, "wrong-git"),
+ GIT_LITERAL_PATHSPECS: "0",
+ LD_PROFILE: "",
+ LD_PRELOAD: undefined,
+ INVENTORY_CALLER_VALUE: "preserved",
+ },
+ },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(marker, "utf8")).toBe("executed");
+ expect(readFileSync(f.out, "utf8")).toBe("./visible.py\n");
+ },
+ );
+
+test.skipIf(process.platform === "win32")(
+ "Git discovery distinguishes absent and empty PATH",
+ () => {
+ const f = fixture();
+ f.write(".gitignore", "retained.txt\n");
+ f.write("retained.txt");
+ git(f.repo, "add", "--force", "retained.txt");
+ const env = {
+ ...process.env,
+ CODEX_SECURITY_GIT: undefined,
+ PATH: undefined,
+ };
+ const result = f.run("make-repo-scope-input", f.scopeArgs(), env);
+ expect(result.status, result.stderr).toBe(0);
+ expect(
+ readFileSync(f.out, "utf8")
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => JSON.parse(line).path),
+ ).toContain("retained.txt");
+ expect(
+ f.run("make-repo-scope-input", f.scopeArgs(), { ...env, PATH: "" })
+ .status,
+ ).not.toBe(0);
+ },
+);
+
+for (const setting of [
+ "GIT_CONFIG_GLOBAL",
+ "GIT_CONFIG_SYSTEM",
+ "XDG_CONFIG_HOME",
+ "GIT_CONFIG_PARAMETERS",
+ "GIT_CONFIG_KEY_0",
+ "GIT_CONFIG_VALUE_0",
+ "RIPGREP_CONFIG_PATH",
+])
+ test.skipIf(process.platform === "win32")(
+ `inventory tools retain raw ${setting} configuration`,
+ () => {
+ const f = fixture();
+ f.write("visible.py");
+ f.write("hidden.py");
+ const rawPath = Buffer.concat([
+ Buffer.from(f.root + "/configuration-"),
+ pathNameBytes,
+ ]);
+ const ignore = join(f.root, "ignore");
+ writeFileSync(ignore, "hidden.py\n");
+ let value = rawPath;
+ if (setting === "GIT_CONFIG_KEY_0") {
+ writeFileSync(
+ join(f.repo, ".git", "HEAD"),
+ Buffer.concat([
+ Buffer.from("ref: refs/heads/branch-"),
+ pathNameBytes,
+ Buffer.from("\n"),
+ ]),
+ );
+ writeFileSync(
+ join(f.root, "config"),
+ `[core]\nexcludesFile = ${JSON.stringify(ignore)}\n`,
+ );
+ value = Buffer.concat([
+ Buffer.from("includeIf.onbranch:branch-"),
+ pathNameBytes,
+ Buffer.from(".path"),
+ ]);
+ } else if (
+ setting === "GIT_CONFIG_PARAMETERS" ||
+ setting === "GIT_CONFIG_VALUE_0"
+ ) {
+ writeFileSync(rawPath, "hidden.py\n");
+ if (setting === "GIT_CONFIG_PARAMETERS")
+ value = Buffer.concat([
+ Buffer.from("'core.excludesFile="),
+ rawPath,
+ Buffer.from("'"),
+ ]);
+ } else {
+ let config = rawPath;
+ if (setting === "XDG_CONFIG_HOME") {
+ mkdirSync(Buffer.concat([rawPath, Buffer.from("/git")]), {
+ recursive: true,
+ });
+ config = Buffer.concat([rawPath, Buffer.from("/git/config")]);
+ }
+ writeFileSync(
+ config,
+ setting === "RIPGREP_CONFIG_PATH"
+ ? "--glob\n!hidden.py\n"
+ : `[core]\nexcludesFile = ${JSON.stringify(ignore)}\n`,
+ );
+ }
+ const octal = [...value]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ const command =
+ setting === "RIPGREP_CONFIG_PATH"
+ ? "generate-in-scope-files"
+ : "make-repo-scope-input";
+ const result = spawnSync(
+ "/bin/sh",
+ [
+ "-c",
+ `${setting}=$(printf '%b' '${octal}'); export ${setting}\nexec "$1" "$2" "$3" --repo "$4" --out "$5" "$6" "$7"`,
+ "inventory-config",
+ node,
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ command,
+ f.repo,
+ f.out,
+ ...(command === "generate-in-scope-files"
+ ? ["--scope", "."]
+ : f.scopeArgs()),
+ ],
+ {
+ encoding: "utf8",
+ env: {
+ ...f.toolEnvironment,
+ ...(setting === "RIPGREP_CONFIG_PATH" ? {} : { HOME: f.root }),
+ CODEX_SECURITY_GIT: Bun.which("git")!,
+ GIT_CONFIG_GLOBAL:
+ setting === "XDG_CONFIG_HOME" ? undefined : "/dev/null",
+ GIT_CONFIG_SYSTEM: "/dev/null",
+ GIT_CONFIG_PARAMETERS: undefined,
+ GIT_CONFIG_COUNT:
+ setting === "GIT_CONFIG_KEY_0" || setting === "GIT_CONFIG_VALUE_0"
+ ? "1"
+ : undefined,
+ GIT_CONFIG_KEY_0: "core.excludesFile",
+ GIT_CONFIG_VALUE_0:
+ setting === "GIT_CONFIG_KEY_0"
+ ? join(f.root, "config")
+ : undefined,
+ GIT_CONFIG_NOSYSTEM: undefined,
+ XDG_CONFIG_HOME: undefined,
+ RIPGREP_CONFIG_PATH: undefined,
+ },
+ },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ const lines = readFileSync(f.out, "utf8").trim().split("\n");
+ expect(
+ setting === "RIPGREP_CONFIG_PATH"
+ ? lines
+ : lines.map((line) => JSON.parse(line).path),
+ ).toEqual([
+ setting === "RIPGREP_CONFIG_PATH" ? "./visible.py" : "visible.py",
+ ]);
+ },
+ );
+
+test("absolute file scopes work when the repository is a filesystem root", () => {
+ const f = fixture();
+ const source = f.write("source.py");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify([source]));
+ const root = parse(f.repo).root;
+ const result = f.run("make-repo-scope-input", [
+ "--repo",
+ root,
+ "--scopes-file",
+ scopes,
+ ]);
+ expect(result.status, result.stderr).toBe(0);
+ expect(JSON.parse(readFileSync(f.out, "utf8")).path).toBe(
+ relative(root, source).split(sep).join("/"),
+ );
+});
+
+test.skipIf(process.platform !== "win32")(
+ "drive-relative scopes stay anchored to the repository",
+ () => {
+ const f = fixture();
+ f.write("src/source.py");
+ const scope = `${parse(f.repo).root.slice(0, 2)}src`;
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify([scope]));
+ expect(
+ f
+ .rows("make-repo-scope-input", ["--scopes-file", scopes])
+ .map((row) => row.path),
+ ).toEqual(["src/source.py"]);
+ },
+);
+
+test("preview trimming handles a full sample of leading blank lines", () => {
+ const f = fixture();
+ f.write("source.py", "\n".repeat(64_000) + "kept\n");
+ expect(f.localRows()[0]!.preview).toBe("kept");
+});
+
+test.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ "scoped inventory skips unreadable descendants of a tracked file replaced by a directory",
+ () => {
+ const f = fixture();
+ f.write("ordinary.py");
+ f.write("replacement");
+ f.commit();
+ rmSync(join(f.repo, "replacement"));
+ f.write("replacement/readable.py");
+ f.write("replacement/blocked/private.py");
+ const blocked = join(f.repo, "replacement", "blocked");
+ chmodSync(blocked, 0);
+ try {
+ expect(f.rows().map((row) => row.path)).toEqual([
+ "ordinary.py",
+ "replacement/readable.py",
+ ]);
+ } finally {
+ chmodSync(blocked, 0o700);
+ }
+ },
+);
+
+test.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ "make-repo-scope-input reports unreadable tracked directories without replacing output",
+ () => {
+ const f = fixture();
+ f.write("private/source.py");
+ git(f.repo, "add", ".");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify(["."]));
+ writeFileSync(f.out, "previous\n");
+ const directory = join(f.repo, "private");
+ chmodSync(directory, 0);
+ try {
+ const result = f.run("make-repo-scope-input", ["--scopes-file", scopes]);
+ expect(result.status).toBe(1);
+ expect(result.stderr).toMatch(/EACCES|permission denied/iu);
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ } finally {
+ chmodSync(directory, 0o700);
+ }
+ },
+);
+
+test.skipIf(process.platform === "win32")(
+ "make-repo-scope-input prunes Git metadata in a tracked file replaced by a directory",
+ () => {
+ const f = fixture();
+ f.write("replacement");
+ f.commit();
+ rmSync(join(f.repo, "replacement"));
+ f.write("replacement/source.py");
+ f.write("replacement/.git/private");
+ const metadata = join(f.repo, "replacement", ".git");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["."]');
+ chmodSync(metadata, 0);
+ try {
+ expect(
+ f
+ .rows("make-repo-scope-input", ["--scopes-file", scopes])
+ .map((row) => row.path),
+ ).toEqual(["replacement/source.py"]);
+ } finally {
+ chmodSync(metadata, 0o700);
+ }
+ },
+);
+
+test("inventory skips stale tracked entries after deletion and directory replacement", () => {
+ const f = fixture();
+ f.write("deleted.py");
+ f.write("replaced/source.py");
+ git(f.repo, "add", ".");
+ rmSync(join(f.repo, "deleted.py"));
+ rmSync(join(f.repo, "replaced"), { recursive: true });
+ f.write("replaced");
+ expect(f.rows().map((row) => row.path)).toEqual(["replaced"]);
+});
+
+test.skipIf(process.platform !== "win32")(
+ "tracked files replaced by a junction to the repository are not traversed",
+ () => {
+ const f = fixture();
+ f.write("cycle");
+ f.write("visible.py");
+ git(f.repo, "add", ".");
+ const junction = join(f.repo, "cycle");
+ rmSync(junction);
+ symlinkSync(f.repo, junction, "junction");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["."]');
+ try {
+ expect(
+ f
+ .rows("make-repo-scope-input", ["--scopes-file", scopes])
+ .map((row) => row.path),
+ ).toEqual(["visible.py"]);
+ } finally {
+ rmSync(junction, { recursive: true, force: true });
+ }
+ },
+);
+
+test("absolute explicit scopes ignore dot and empty components before the repo", () => {
+ const f = fixture();
+ f.write("src/source.py");
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(
+ scopes,
+ JSON.stringify([
+ `${f.root}${sep}.${sep}repository${sep}src${sep}source.py`,
+ `${f.root}${sep}${sep}repository${sep}src${sep}source.py`,
+ ]),
+ );
+ expect(
+ f
+ .rows("make-repo-scope-input", ["--scopes-file", scopes])
+ .map((row) => row.path),
+ ).toEqual(["src/source.py"]);
+});
+
+test("diff inventory expands a home-relative repository scope", () => {
+ const f = fixture();
+ f.write("source.py", "before\n");
+ const base = f.commit();
+ f.write("source.py", "after\n");
+ const result = f.run(
+ "generate-in-scope-files",
+ [
+ "--scope",
+ "~/repository",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ "local-patch",
+ ],
+ { ...process.env, HOME: f.root, USERPROFILE: f.root },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(readFileSync(f.out, "utf8")).toBe("source.py\n");
+});
+
+for (const rawHome of [false, true])
+ test.skipIf(process.platform === "win32")(
+ `inventory falls back to ripgrep when selected Git's interpreter is missing${rawHome ? " with raw HOME" : ""}`,
+ () => {
+ const f = fixture();
+ f.write("scope/source.py");
+ const selectedGit = join(f.root, "git");
+ writeFileSync(selectedGit, `#!${join(f.root, "missing-interpreter")}\n`, {
+ mode: 0o700,
+ });
+ const result = runPathInventory(
+ f,
+ "make-repo-scope-input",
+ {
+ ...f.toolEnvironment,
+ CODEX_SECURITY_GIT: selectedGit,
+ },
+ rawHome,
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(JSON.parse(readFileSync(f.out, "utf8")).path).toBe(
+ "scope/source.py",
+ );
+ },
+ );
+
+test("inventory preserves scoped names beneath dotted-I repository names", () => {
+ const f = fixture();
+ f.write("scope/source.py", "kept\n");
+ const repository = join(f.root, "İrepository");
+ renameSync(f.repo, repository);
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify(["scope"]));
+ const rows = f.rows("make-repo-scope-input", [
+ "--repo",
+ repository,
+ "--scopes-file",
+ scopes,
+ ]);
+ expect(rows.map((row) => row.path)).toEqual(["scope/source.py"]);
+});
+
+test.skipIf(process.platform !== "win32")(
+ "Windows Git inventories and revision previews retain indexed lone UTF-16 names",
+ () => {
+ const f = fixture();
+ f.write("base.py");
+ const base = f.commit();
+ const names = ["high-\ud800.py", "low-\udfff.py"];
+ f.write(".gitignore", "high-*\nlow-*\n");
+ git(f.repo, "add", ".gitignore");
+ try {
+ const blob = spawnSync(
+ "git",
+ ["-C", f.repo, "hash-object", "-w", "--stdin"],
+ { input: "kept\n", encoding: "utf8" },
+ );
+ expect(blob.status, blob.stderr).toBe(0);
+ const rawNames = [
+ Buffer.concat([
+ Buffer.from("high-"),
+ Buffer.from([0xed, 0xa0, 0x80]),
+ Buffer.from(".py"),
+ ]),
+ Buffer.concat([
+ Buffer.from("low-"),
+ Buffer.from([0xed, 0xbf, 0xbf]),
+ Buffer.from(".py"),
+ ]),
+ ];
+ const index = Buffer.concat(
+ rawNames.map((name) =>
+ Buffer.concat([
+ Buffer.from(`100644 ${blob.stdout.trim()}\t`),
+ name,
+ Buffer.from([0]),
+ ]),
+ ),
+ );
+ const updated = spawnSync(
+ "git",
+ ["-C", f.repo, "update-index", "-z", "--index-info"],
+ { input: index },
+ );
+ expect(updated.status, updated.stderr.toString()).toBe(0);
+ git(f.repo, "checkout-index", "--all", "--force");
+ git(f.repo, "commit", "-qm", "Indexed path fixture");
+ for (const diff of [false, true]) {
+ const inventory = f.run("generate-in-scope-files", [
+ "--scope",
+ ".",
+ ...(diff ? ["--diff-base", base] : []),
+ ]);
+ expect(inventory.status, inventory.stderr).toBe(0);
+ for (const name of rawNames)
+ expect(
+ readFileSync(f.out).includes(
+ Buffer.concat([
+ Buffer.from(diff ? "" : "./"),
+ name,
+ Buffer.from("\n"),
+ ]),
+ ),
+ ).toBe(true);
+ }
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["."]');
+ for (const command of ["make-repo-scope-input", "make-diff-rank-input"]) {
+ const rows = f.rows(
+ command,
+ command === "make-diff-rank-input"
+ ? ["--base", base]
+ : ["--scopes-file", scopes],
+ );
+ expect(
+ rows
+ .filter((row) => /^(high|low)-/u.test(row.path))
+ .map((row) => row.path),
+ ).toEqual(names);
+ if (command !== "make-repo-scope-input")
+ for (const name of names)
+ expect(rows.find((row) => row.path === name)?.preview).toBe("kept");
+ }
+ } finally {
+ git(f.repo, "reset", "--hard", base);
+ }
+ },
+);
+
+for (const raw of ["ordinary", "HOME", "PATH", "repository"])
+ for (const tool of ["missing", "non-executable"])
+ test.skipIf(process.platform === "win32")(
+ `make-repo-scope-input safely falls back with ${tool} ripgrep for ${raw} POSIX paths`,
+ () => {
+ const f = fixture();
+ rmSync(join(f.repo, ".git"), { recursive: true });
+ f.write("scope/visible.py", "source\n");
+ f.write("outside.py");
+ const repo =
+ raw === "repository"
+ ? Buffer.concat([Buffer.from(f.root + "/repo-"), pathNameBytes])
+ : Buffer.from(f.repo);
+ if (raw === "repository") renameSync(f.repo, repo);
+ const home = Buffer.concat([
+ Buffer.from(f.root + "/home-"),
+ raw === "HOME" ? Buffer.from([0xff]) : Buffer.from("plain"),
+ ]);
+ const bin = join(f.root, "bin");
+ mkdirSync(bin);
+ if (tool === "non-executable")
+ writeFileSync(join(bin, "rg"), "#!/bin/sh\nexit 0\n", {
+ mode: 0o600,
+ });
+ const path = Buffer.concat([
+ Buffer.from(bin),
+ raw === "PATH" ? Buffer.from([58, 47, 0xff]) : Buffer.alloc(0),
+ ]);
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, '["scope"]');
+ const octal = (bytes: Buffer) =>
+ [...bytes]
+ .map((byte) => `\\0${byte.toString(8).padStart(3, "0")}`)
+ .join("");
+ const script = [
+ `HOME=$(printf '%b' '${octal(home)}'); export HOME`,
+ `PATH=$(printf '%b' '${octal(path)}'); export PATH`,
+ 'exec "$1" "$2" --helper 3<&0 0</dev/null',
+ ].join("\n");
+ const input = Buffer.concat([
+ Buffer.from("1\0"),
+ home,
+ Buffer.from("\0make-repo-scope-input\0--repo\0"),
+ repo,
+ Buffer.from(`\0--out\0${f.out}\0--scopes-file\0${scopes}\0`),
+ ]).toString("hex");
+ const run = () =>
+ spawnSync(
+ "/bin/sh",
+ [
+ "-c",
+ script,
+ "inventory-env",
+ node,
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ ],
+ {
+ encoding: "utf8",
+ input,
+ env: { ...process.env, CODEX_SECURITY_GIT: "" },
+ },
+ );
+ const result = run();
+ expect(result.status, result.stderr).toBe(0);
+ expect(JSON.parse(readFileSync(f.out, "utf8"))).toEqual({
+ path: "scope/visible.py",
+ });
+ writeFileSync(
+ Buffer.concat([repo, Buffer.from("/scope/.ignore")]),
+ "visible.py\n",
+ );
+ writeFileSync(f.out, "previous\n");
+ const ignored = run();
+ expect(ignored.status).toBe(1);
+ expect(ignored.stderr).toContain("without Git or ripgrep");
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ },
+ );
+
+test.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ "missing-tool fallback retains readable files beside unreadable directories",
+ () => {
+ const f = fixture();
+ rmSync(join(f.repo, ".git"), { recursive: true });
+ f.write("readable.py");
+ f.write("blocked/private.py");
+ const blocked = join(f.repo, "blocked");
+ chmodSync(blocked, 0);
+ try {
+ const result = f.run("make-repo-scope-input", f.scopeArgs(), {
+ ...process.env,
+ CODEX_SECURITY_GIT: "",
+ PATH: f.root,
+ });
+ expect(result.status, result.stderr).toBe(0);
+ expect(JSON.parse(readFileSync(f.out, "utf8"))).toEqual({
+ path: "readable.py",
+ });
+ } finally {
+ chmodSync(blocked, 0o700);
+ }
+ },
+);
+
+for (const name of [".gitignore", ".ignore", ".rgignore"])
+ for (const location of ["ancestor", "descendant"])
+ test.skipIf(process.platform === "win32")(
+ `missing-tool fallback notices ${location} symlinked ${name}`,
+ () => {
+ const f = fixture();
+ rmSync(join(f.repo, ".git"), { recursive: true });
+ f.write("scope/nested/source.py");
+ const rules = join(f.root, "ignore-rules");
+ writeFileSync(rules, "source.py\n");
+ symlinkSync(
+ rules,
+ join(f.repo, location === "ancestor" ? name : `scope/nested/${name}`),
+ );
+ writeFileSync(f.out, "previous\n");
+ const result = f.run("make-repo-scope-input", f.scopeArgs("scope"), {
+ ...process.env,
+ CODEX_SECURITY_GIT: "",
+ PATH: f.root,
+ });
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain("without Git or ripgrep");
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ },
+ );
+
+for (const marker of [".git", ".gitignore"])
+ test.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ `missing-tool fallback rejects ${marker} before reading descendants`,
+ () => {
+ const f = fixture();
+ const scope = marker === ".git" ? "." : "scope";
+ const blocked =
+ marker === ".git"
+ ? join(f.repo, ".git", "objects")
+ : join(f.repo, scope, "unreadable");
+ if (marker === ".gitignore") {
+ rmSync(join(f.repo, ".git"), { recursive: true });
+ f.write(".gitignore", "ignored.py\n");
+ f.write("scope/unreadable/source.py");
+ }
+ writeFileSync(f.out, "previous\n");
+ chmodSync(blocked, 0);
+ try {
+ const result = f.run("make-repo-scope-input", f.scopeArgs(scope), {
+ ...process.env,
+ CODEX_SECURITY_GIT: "",
+ PATH: f.root,
+ });
+ expect(result.status).toBe(1);
+ expect(result.stderr.trim()).toBe(
+ "Could not safely enumerate ignored scoped files without Git or ripgrep.",
+ );
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ } finally {
+ chmodSync(blocked, 0o700);
+ }
+ },
+ );
+
+for (const name of [".gitignore", ".ignore", ".rgignore"])
+ test.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ `missing-tool fallback finds descendant ${name} before entering sibling directories`,
+ () => {
+ const f = fixture();
+ rmSync(join(f.repo, ".git"), { recursive: true });
+ f.write(`nested/${name}`, "ignored.py\n");
+ f.write("nested/!unreadable/source.py");
+ const blocked = join(f.repo, "nested", "!unreadable");
+ writeFileSync(f.out, "previous\n");
+ chmodSync(blocked, 0);
+ try {
+ const result = f.run("make-repo-scope-input", f.scopeArgs(), {
+ ...process.env,
+ CODEX_SECURITY_GIT: "",
+ PATH: f.root,
+ });
+ expect(result.status).toBe(1);
+ expect(result.stderr.trim()).toBe(
+ "Could not safely enumerate ignored scoped files without Git or ripgrep.",
+ );
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ } finally {
+ chmodSync(blocked, 0o700);
+ }
+ },
+ );
+
+test("file and Git previews retain empty files and incomplete final UTF-16 units", () => {
+ const f = fixture();
+ f.write("base.py");
+ const base = f.commit();
+ f.write("empty.py", "");
+ f.write("utf16.py", Buffer.from([0xff, 0xfe, 0]));
+ f.commit();
+ for (const rows of [
+ f.localRows(),
+ f.rows("make-diff-rank-input", ["--base", base]),
+ ]) {
+ expect(rows.find((row) => row.path === "empty.py")?.preview).toBe("");
+ expect(rows.find((row) => row.path === "utf16.py")?.preview).toBe("");
+ }
+});
+
+for (const signal of ["TERM", "KILL"])
+ for (const command of ["generate-in-scope-files", "make-repo-scope-input"])
+ test.skipIf(process.platform === "win32")(
+ `${command} preserves output when ripgrep is terminated by SIG${signal}`,
+ () => {
+ const f = fixture();
+ f.write("partial.py");
+ f.write("omitted.py");
+ const bin = join(f.root, "bin");
+ mkdirSync(bin);
+ writeFileSync(
+ join(bin, "rg"),
+ `#!/bin/sh\nprintf './partial.py\\0'\nkill -${signal} $$\n`,
+ { mode: 0o700 },
+ );
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify(["."]));
+ writeFileSync(f.out, "previous\n");
+ const result = f.run(
+ command,
+ command === "make-repo-scope-input"
+ ? ["--scopes-file", scopes]
+ : ["--scope", "."],
+ { ...process.env, CODEX_SECURITY_GIT: "", PATH: bin },
+ );
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toContain(`SIG${signal}`);
+ expect(readFileSync(f.out, "utf8")).toBe("previous\n");
+ },
+ );
+
+for (const selectedPath of [".", "component"]) {
+ for (const linkedWorktree of [false, true]) {
+ test(`inventory retains ignored tracked paths in ${selectedPath}, linked=${linkedWorktree}`, () => {
+ const f = fixture();
+ f.write(".gitignore", "ignored/\n");
+ f.commit();
+ let repository = f.repo;
+ if (linkedWorktree) {
+ repository = join(f.root, "worktree");
+ git(f.repo, "worktree", "add", "-q", "--detach", repository, "HEAD");
+ }
+ const selected = join(repository, selectedPath);
+ mkdirSync(join(selected, "ignored"), { recursive: true });
+ writeFileSync(
+ join(selected, "ignored", "tracked.py"),
+ "tracked = True\n",
+ );
+ writeFileSync(
+ join(selected, "ignored", "untracked.py"),
+ "untracked = True\n",
+ );
+ git(selected, "add", "--force", "--", "ignored/tracked.py");
+ const result = f.run("generate-in-scope-files", [
+ "--repo",
+ selected,
+ "--scope",
+ ".",
+ ]);
+ expect(result.status, result.stderr).toBe(0);
+ const paths = readFileSync(f.out, "utf8").split("\n");
+ expect(paths).toContain("./ignored/tracked.py");
+ expect(paths).not.toContain("./ignored/untracked.py");
+ });
+ }
+}
+
+for (const scope of [".", ".svn", "vendor/.svn/pristine/base"]) {
+ test(`inventory excludes tracked SVN metadata for ${scope}`, () => {
+ const f = fixture();
+ f.write(".svn/wc.db");
+ f.write("vendor/.svn/pristine/base");
+ f.write("source.py");
+ f.write(".gitignore", ".svn/\n");
+ git(f.repo, "add", "--force", ".svn", "vendor/.svn");
+ const paths = f
+ .success("generate-in-scope-files", ["--scope", scope])
+ .trim()
+ .split("\n")
+ .filter(Boolean);
+ expect(paths.every((path) => !path.split("/").includes(".svn"))).toBe(true);
+ if (scope === ".") expect(paths).toContain("./source.py");
+ else expect(paths).toEqual([]);
+ });
+}
+
+test("scope input excludes explicitly requested SVN metadata", () => {
+ const f = fixture();
+ f.write("src/source.py");
+ f.write("src/.svn/config");
+ f.commit();
+ const scopes = join(f.root, "scopes.json");
+ writeFileSync(scopes, JSON.stringify(["src", "src/.svn/config"]));
+ expect(
+ f
+ .rows("make-repo-scope-input", ["--scopes-file", scopes])
+ .map((row) => row.path),
+ ).toEqual(["src/source.py"]);
+});
+
+test("diff inventory accepts a base revision matching a path", () => {
+ const f = fixture();
+ f.write("release/build.py", "build = 1\n");
+ f.commit();
+ git(f.repo, "branch", "release");
+ f.write("release/build.py", "build = 2\n");
+ expect(
+ f
+ .rows("make-diff-rank-input", [
+ "--base",
+ "release",
+ "--mode",
+ "local-patch",
+ ])
+ .map((row) => row.path),
+ ).toEqual(["release/build.py"]);
+});
+
+test("local diffs exclude SVN metadata and revision diffs retain it", () => {
+ const f = fixture();
+ const paths = (state: string) => [
+ state + ".py",
+ "vendor/.svn-" + state + ".py",
+ ".svn/" + state,
+ "vendor/.svn/" + state,
+ ];
+ const states = ["staged", "unstaged", "untracked", "deleted"];
+ for (const state of ["staged", "unstaged", "deleted"]) {
+ for (const name of paths(state)) f.write(name, name + ": before\n");
+ }
+ const base = f.commit();
+ for (const state of ["staged", "unstaged", "untracked"]) {
+ for (const name of paths(state)) f.write(name, name + ": after\n");
+ }
+ git(f.repo, "add", "--", ...paths("staged"));
+ for (const name of paths("deleted")) rmSync(join(f.repo, name));
+ for (const mode of ["local-patch", "revisions"]) {
+ if (mode === "revisions") f.commit();
+ const expected = states
+ .flatMap((state) =>
+ mode === "revisions" ? paths(state) : paths(state).slice(0, 2),
+ )
+ .sort();
+ expect(
+ f
+ .rows("make-diff-rank-input", ["--base", base, "--mode", mode])
+ .map((row) => row.path),
+ ).toEqual(expected);
+ expect(
+ f
+ .success("generate-in-scope-files", [
+ "--scope",
+ ".",
+ "--diff-base",
+ base,
+ "--diff-mode",
+ mode,
+ ])
+ .trim()
+ .split("\n"),
+ ).toEqual(expected);
+ }
+});
+
+test("inventory helpers print their usage without a repository", () => {
+ for (const command of [
+ "generate-in-scope-files",
+ "make-repo-scope-input",
+ "make-diff-rank-input",
+ ]) {
+ const result = spawnSync(
+ node,
+ [join(PLUGIN_ROOT, "mcp", "helpers.mjs"), command, "--help"],
+ { encoding: "utf8" },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain(`--helper ${command}`);
+ }
+});
+
+test("keeps the retired repository-ranking helper unavailable", () => {
+ const result = fixture().run("make-repo-rank-input", ["--help"]);
+ expect(result.status).toBe(2);
+ expect(result.stderr).not.toContain("make-repo-rank-input");
+});
diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts
index e47c7fb1e..cae6ac713 100644
--- a/sdk/typescript/tests-ts/runtime.test.ts
+++ b/sdk/typescript/tests-ts/runtime.test.ts
@@ -1,4 +1,5 @@
import { runNodePython } from "./support/python-probe.js";
+import { nodeCommand } from "./support/shell.js";
import { scanRegistrationArguments } from "./support/workbench-command.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { parseJsonLines, jsonLines } from "./support/json.js";
@@ -422,21 +423,27 @@ describe("plugin runtime preparation", () => {
expect(initialized.status, initialized.stderr).toBe(0);
}
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
const output = join(root, "inventory.txt");
const repeatedOutput = join(root, "inventory-repeated.txt");
- const generatorArguments = (destination: string) => [
- join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"),
- "--repo",
- repository,
- "--scope",
- ".",
- "--out",
- destination,
- ];
+ const generatorArguments = (destination: string) =>
+ [
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "generate-in-scope-files",
+ "--repo",
+ repository,
+ "--scope",
+ ".",
+ "--out",
+ destination,
+ ] as const;
for (const destination of [output, repeatedOutput]) {
- const inventory = runNodePython(python!, generatorArguments(destination));
+ const inventory = spawnSync(
+ nodeCommand().command,
+ generatorArguments(destination),
+ {
+ encoding: "utf8",
+ },
+ );
expect(inventory.status, inventory.stderr).toBe(0);
}
@@ -462,7 +469,13 @@ describe("plugin runtime preparation", () => {
);
await writeFile(join(repository, "literal:colon.txt"), "colon\n");
const posixOutput = join(root, "inventory-posix-filenames.txt");
- const inventory = runNodePython(python!, generatorArguments(posixOutput));
+ const inventory = spawnSync(
+ nodeCommand().command,
+ generatorArguments(posixOutput),
+ {
+ encoding: "utf8",
+ },
+ );
expect(inventory.status, inventory.stderr).toBe(0);
const posixRows = (await readFile(posixOutput, "utf8"))
.trimEnd()
@@ -486,34 +499,39 @@ describe("plugin runtime preparation", () => {
join(repository, "source.ts:synthetic-stream"),
"export const hidden = true;\n",
);
- const python =
- process.env["PYTHON"] ?? Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
-
- const inventory = runNodePython(python!, [
- join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"),
- "--repo",
- repository,
- "--scope",
- "source.ts:synthetic-stream",
- "--out",
- join(root, "inventory.txt"),
- ]);
+ const inventory = spawnSync(
+ nodeCommand().command,
+ [
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "generate-in-scope-files",
+ "--repo",
+ repository,
+ "--scope",
+ "source.ts:synthetic-stream",
+ "--out",
+ join(root, "inventory.txt"),
+ ],
+ { encoding: "utf8" },
+ );
expect(inventory.status).toBe(2);
- expect(inventory.stderr).toContain("NTFS alternate data streams");
+ expect(inventory.stderr).toContain("NTFS alternate data stream");
const scopes = join(root, "scopes.json");
await writeFile(scopes, JSON.stringify(["source.ts:synthetic-stream"]));
- const scopeInput = runNodePython(python!, [
- join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),
- "make-repo-scope-input",
- "--repo",
- repository,
- "--scopes-file",
- scopes,
- "--out",
- join(root, "scope-input.jsonl"),
- ]);
+ const scopeInput = spawnSync(
+ nodeCommand().command,
+ [
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
+ "make-repo-scope-input",
+ "--repo",
+ repository,
+ "--scopes-file",
+ scopes,
+ "--out",
+ join(root, "scope-input.jsonl"),
+ ],
+ { encoding: "utf8" },
+ );
expect(scopeInput.status).toBe(1);
expect(scopeInput.stderr).toContain("NTFS alternate data stream");
},
@@ -6272,23 +6290,20 @@ describe("runtime directories and plugin Python boundary", () => {
});
test.skipIf(process.platform !== "win32")(
- "runs plugin helpers with UTF-8 standard streams",
+ "runs Node inventory helpers with UTF-8 standard streams",
async () => {
const root = await temporaryDirectory("codex-security-python-utf8-");
const repository = join(root, "repository");
const output = join(root, "出力.jsonl");
await mkdir(repository);
await writeFile(join(repository, "source.py"), "value = 1\n");
- const python = Bun.which("python3") ?? Bun.which("python");
- expect(python).not.toBeNull();
const scopes = join(root, "scopes.json");
await writeFile(scopes, JSON.stringify(["."]));
const result = spawnSync(
- python!,
+ nodeCommand().command,
[
- "-B",
- join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),
+ join(PLUGIN_ROOT, "mcp", "helpers.mjs"),
"make-repo-scope-input",
"--repo",
repository,
@@ -6299,10 +6314,11 @@ describe("runtime directories and plugin Python boundary", () => {
],
{
encoding: "utf8",
- env: pluginExecutionEnvironment(python!, {
+ env: {
...process.env,
pythonutf8: "0",
- }),
+ PYTHON: "missing-python",
+ },
},
);
diff --git a/sdk/typescript/tests-ts/windows-helper-command.ts b/sdk/typescript/tests-ts/windows-helper-command.ts
index e793a3bac..bb943f72d 100644
--- a/sdk/typescript/tests-ts/windows-helper-command.ts
+++ b/sdk/typescript/tests-ts/windows-helper-command.ts
@@ -41,9 +41,12 @@ export function windowsHelperFixture(
input?: string,
workingDirectory = root,
pipelineInput?: string,
+ blockIndex = 0,
) {
const source = readFileSync(join(PLUGIN_ROOT, document), "utf8");
- let command = /```powershell\r?\n([\s\S]*?)\r?\n```/u.exec(source)?.[1];
+ let command = [
+ ...source.matchAll(/```powershell\r?\n([\s\S]*?)\r?\n```/gu),
+ ][blockIndex]?.[1];
if (command === undefined)
throw new Error(`No PowerShell command in ${document}`);
for (const [placeholder, path] of Object.entries(paths))
diff --git a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts
index b2564e6dd..e10bfff4a 100644
--- a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts
+++ b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts
@@ -1,3 +1,4 @@
+import { nodeCommand } from "./support/shell.js";
import { chmod, mkdir, realpath, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, dirname, join } from "node:path";
@@ -150,7 +151,7 @@ describe("bundled workbench canonical paths", () => {
"for selected, original, expected in cases:",
" assert selected.samefile(original)",
" assert target.directory_content_digest(selected) == target.directory_content_digest(original)",
- " subprocess.run([sys.executable, str(Path(sys.argv[1]) / 'generate_rank_input.py'), 'make-repo-scope-input', '--repo', str(selected), '--scopes-file', str(scopes), '--out', str(output)], check=True, capture_output=True)",
+ ` subprocess.run([${JSON.stringify(nodeCommand().command)}, str(Path(sys.argv[1]).parent / 'mcp' / 'helpers.mjs'), 'make-repo-scope-input', '--repo', str(selected), '--scopes-file', str(scopes), '--out', str(output)], check=True, capture_output=True)`,
" assert [json.loads(line)['path'] for line in output.read_text().splitlines()] == expected",
"print(json.dumps({'targetsChecked': len(cases)}))",
].join("\n"),