OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
refactor(plugin)!: retire obsolete ranking helpers (#1607)
Michael D'Angelof894583a3b8d
diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.tsindex c0c352aed..ec7f32402 100644--- a/plugins/codex-security/mcp-app/helpers-main.ts+++ b/plugins/codex-security/mcp-app/helpers-main.ts@@ -15,8 +15,6 @@ import {} from "./src/helpers/normalize-candidates";import { validatePatchRiskAssessmentCommand } from "./src/helpers/validate-patch-risk-assessment";import { deepReviewInputCommand } from "./src/helpers/deep-review-input";-import { rankShardsCommand } from "./src/helpers/rank-shards";-import { rankPoolCommand } from "./src/helpers/rank-pool";import { bindRepoScopesCommand } from "./src/helpers/bind-repo-scopes";import { escapeControls, stringifyJson } from "./src/helpers/json";import { decodeUtf8 } from "./src/helpers/utf8";@@ -84,23 +82,8 @@ function runHelper(): void {process.exitCode = normalizeCandidatesCommand(args, posixHome);} else if (command === "validate-patch-risk-assessment") {process.exitCode = validatePatchRiskAssessmentCommand(args);- } else if (- command === "copy-deep-review-input" ||- command === "select-deep-review-input"- ) {- process.exitCode = deepReviewInputCommand(command, args, posixHome);- } else if (- command === "make-rank-shards" ||- command === "validate-rank-shard" ||- command === "merge-rank-outputs"- ) {- process.exitCode = rankShardsCommand(command, args, posixHome);- } else if (- command === "make-rank-pool-plan" ||- command === "validate-rank-worker" ||- command === "validate-rank-pool"- ) {- process.exitCode = rankPoolCommand(command, args, posixHome);+ } else if (command === "copy-deep-review-input") {+ process.exitCode = deepReviewInputCommand(args, posixHome);} else if (command === "bind-repo-scopes") {process.exitCode = bindRepoScopesCommand(args, posixHome);} else if (Object.hasOwn(workbenchUsage, command)) {@@ -149,7 +132,7 @@ function runHelper(): void {});} else {console.error(- `Usage: launch_codex_security_mcp[.cmd] --helper <resolve-security-md | normalize-candidates | validate-patch-risk-assessment | copy-deep-review-input | select-deep-review-input | make-rank-shards | validate-rank-shard | merge-rank-outputs | make-rank-pool-plan | validate-rank-worker | validate-rank-pool | bind-repo-scopes | ${Object.keys(workbenchUsage).join(" | ")}> [options]`,+ `Usage: launch_codex_security_mcp[.cmd] --helper <resolve-security-md | normalize-candidates | validate-patch-risk-assessment | copy-deep-review-input | bind-repo-scopes | ${Object.keys(workbenchUsage).join(" | ")}> [options]`,);process.exitCode = 2;}diff --git a/plugins/codex-security/mcp-app/src/helpers/deep-review-input.ts b/plugins/codex-security/mcp-app/src/helpers/deep-review-input.tsindex cb04e62ec..1ff2ec280 100644--- a/plugins/codex-security/mcp-app/src/helpers/deep-review-input.ts+++ b/plugins/codex-security/mcp-app/src/helpers/deep-review-input.ts@@ -1,6 +1,5 @@import {argumentsFor,- compare,loadRankRows,print,reportCommandError,@@ -9,55 +8,30 @@ import {writeRankRows,} from "./rank-worklists";-type Command = "copy-deep-review-input" | "select-deep-review-input";-export function deepReviewInputCommand(- command: Command,args: string[],posixHome = process.env.HOME,): number {- const selection = command === "select-deep-review-input";- const input = selection ? "rank-output" : "rank-input";- const usage = `usage: launch_codex_security_mcp[.cmd] --helper ${command} [-h] --${input} PATH --out PATH${selection ? " [--top-percent INT]" : ""}`;+ const command = "copy-deep-review-input";+ const usage = `usage: launch_codex_security_mcp[.cmd] --helper ${command} [-h] --rank-input PATH --out PATH`;try {- const values = argumentsFor(- args,- [input, "out"],- selection ? ["top-percent"] : [],- );+ const values = argumentsFor(args, ["rank-input", "out"]);if (values.help) {print(- `${usage}\n\nCreate deep_review_input.jsonl from ${selection ? "worker-produced rank_output.jsonl" : "rank_input.jsonl"}.\n\noptions:\n -h, --help show this help message and exit\n --${input} PATH ${selection ? "Worker ranking output" : "Deterministic rank input"} JSONL.\n --out PATH Output deep_review_input.jsonl path.${selection ? "\n --top-percent INT Percent of included files to keep for deep review. Defaults to 100." : ""}`,+ `${usage}\n\nCreate deep_review_input.jsonl from rank_input.jsonl.\n\noptions:\n -h, --help show this help message and exit\n --rank-input PATH Deterministic rank input JSONL.\n --out PATH Output deep_review_input.jsonl path.`,);return 0;}- const path = (name: string) =>- worklistPath(values[name] as string, posixHome);- const rows = loadRankRows(path(input), selection);- requireUniquePaths(rows, selection ? "Rank output" : "Rank input");- let selected = rows,- total = rows.length;- if (selection) {- const included = rows.filter((row) => row.include);- const base = included.length ? included : rows;- base.sort(- (left, right) =>- right.score! - left.score! || compare(left.path, right.path),- );- total = base.length;- const percent = (values["top-percent"] ?? 100) as number;- const keep = total ? Math.max(1, Math.trunc(total * (percent / 100))) : 0;- selected = base.slice(0, keep);- }- const output = path("out");+ const rows = loadRankRows(+ worklistPath(values["rank-input"] as string, posixHome),+ );+ requireUniquePaths(rows, "Rank input");+ const output = worklistPath(values.out as string, posixHome);writeRankRows(output,- selected.map(({ path, area }) => ({ path, area })),+ rows.map(({ path, area }) => ({ path, area })),);- const message = selection- ? `Selected ${selected.length} of ${total} rows into ${output}`- : `Copied ${selected.length} rows into ${output}`;- print(message);+ print(`Copied ${rows.length} rows into ${output}`);return 0;} catch (error) {return reportCommandError(error, command, usage);diff --git a/plugins/codex-security/mcp-app/src/helpers/rank-pool.ts b/plugins/codex-security/mcp-app/src/helpers/rank-pool.tsdeleted file mode 100644index f7f4cb558..000000000--- a/plugins/codex-security/mcp-app/src/helpers/rank-pool.ts+++ /dev/null@@ -1,329 +0,0 @@-import { isDeepStrictEqual as same } from "node:util";-import { createHash } from "node:crypto";-import { basename, dirname } from "node:path";-import { mkdir, readFile, writeFile } from "./helper-files";-import { formatDiagnostic, object, parseJson } from "./json";-import { decodeUtf8 } from "./utf8";-import { encodePosixPath } from "./posix-path";-import { resolvedPath } from "./resolve-security-md";-import {- childPath,- discoverInputShards,- outputShardName,- shardNames,- validateShard,-} from "./rank-shards";-import {- argumentsFor,- compare,- missingNames,- print,- reportCommandError,- worklistPath,-} from "./rank-worklists";--type Command =- "make-rank-pool-plan" | "validate-rank-worker" | "validate-rank-pool";-interface Worker {- input_shards: string[];- output_shards: string[];- slot: number;-}--function requirePlanDirectory(plan: string, directory: string): void {- const expected = childPath(dirname(plan), "rank_shards");- const key = (value: string) =>- process.platform === "win32"- ? resolvedPath(Buffer.from(value, "utf16le"), false)- .toString("utf16le")- .toLowerCase()- : resolvedPath(encodePosixPath(value), false).toString("latin1");- if (key(directory) !== key(expected))- throw new Error(- `Rank shard directory must be the assignment plan's sibling rank_shards directory; expected=${expected}; actual=${directory}`,- );-}--function makePlan(directory: string, slots: number, output: string): void {- requirePlanDirectory(output, directory);- const inputs = discoverInputShards(directory).map((path) => basename(path));- const count = Math.min(inputs.length, slots, 6);- const workers = Array.from({ length: count }, (_, index): Worker => {- const assigned = inputs.filter(- (_, inputIndex) => inputIndex % count === index,- );- return {- input_shards: assigned,- output_shards: assigned.map(outputShardName),- slot: index + 1,- };- });- const plan = {- ranking_worker_count: count,- schema_version: 1,- shard_count: inputs.length,- strategy: "round_robin",- workers,- };- mkdir(dirname(output));- writeFile(output, [- Buffer.from(- (JSON.stringify(plan, null, 2) + "\n").replaceAll(- "\n",- process.platform === "win32" ? "\r\n" : "\n",- ),- ),- ]);- print(- `Assigned ${inputs.length} rank shards to ${count} ranking workers in ${output}`,- );-}--function integer(value: unknown, label: string, minimum: number): number {- if (typeof value !== "number" || !Number.isInteger(value) || value < minimum)- throw new Error(`${label} must be an integer of at least ${minimum}`);- return value;-}--function strings(value: unknown, label: string): string[] {- if (!Array.isArray(value) || value.length === 0)- throw new Error(`${label} must be a non-empty list`);- if (value.some((item: unknown) => typeof item !== "string" || !item))- throw new Error(`${label} entries must be non-empty strings`);- return value as string[];-}--function fields(- value: Record<string, unknown>,- expected: string[],- label: string,-): void {- const actual = Object.keys(value);- const missing = missingNames(expected, actual);- const unexpected = missingNames(actual, expected);- if (missing.length || unexpected.length)- throw new Error(- `${label} fields do not match schema; missing=${formatDiagnostic(missing)}; unexpected=${formatDiagnostic(unexpected)}`,- );-}--function validatePlan(plan: string, directory: string) {- requirePlanDirectory(plan, directory);- const misplaced = [- ...shardNames(dirname(plan), "input"),- ...shardNames(dirname(plan), "output"),- ].sort(compare);- if (misplaced.length)- throw new Error(- `Rank shard artifacts must be stored in the assignment plan's sibling rank_shards directory; misplaced=${formatDiagnostic(misplaced)}`,- );- const inputs = discoverInputShards(directory);- const inputNames = inputs.map((path) => basename(path));- const outputNames = inputNames.map(outputShardName);- const bytes = readFile(plan, "Rank pool plan");- let payload: unknown;- try {- payload = parseJson(decodeUtf8(bytes).replace(/^\uFEFF/u, ""));- } catch (error) {- if (error instanceof SyntaxError)- throw new Error(`${plan}: invalid JSON: ${error.message}`);- throw error;- }- if (!object(payload)) throw new Error(`${plan}: expected a JSON object`);- fields(- payload,- [- "schema_version",- "strategy",- "shard_count",- "ranking_worker_count",- "workers",- ],- `${plan}: rank pool plan`,- );- if (integer(payload.schema_version, `${plan}: schema_version`, 1) !== 1)- throw new Error(`${plan}: schema_version must be 1`);- if (payload.strategy !== "round_robin")- throw new Error(`${plan}: strategy must be round_robin`);- const shardCount = integer(payload.shard_count, `${plan}: shard_count`, 0);- if (shardCount !== inputs.length)- throw new Error(- `${plan}: shard_count does not match input shards; plan=${shardCount}; actual=${inputs.length}`,- );- const workerCount = integer(- payload.ranking_worker_count,- `${plan}: ranking_worker_count`,- 0,- );- if (shardCount > 0 && workerCount === 0)- throw new Error(- `${plan}: ranking_worker_count must be at least 1 when input shards exist`,- );- if (workerCount > shardCount)- throw new Error(`${plan}: ranking_worker_count cannot exceed shard_count`);- if (workerCount > 6)- throw new Error(`${plan}: ranking_worker_count cannot exceed 6`);- if (!Array.isArray(payload.workers) || payload.workers.length !== workerCount)- throw new Error(- `${plan}: workers must contain exactly ${workerCount} worker assignments`,- );- payload.workers.forEach((raw: unknown, index) => {- const label = `${plan}: workers[${index}]`;- if (!object(raw)) throw new Error(`${label} must be a JSON object`);- fields(raw, ["slot", "input_shards", "output_shards"], label);- const slot = integer(raw.slot, `${label}.slot`, 1);- if (slot !== index + 1)- throw new Error(`${label}.slot must be ${index + 1}`);- const assignedInputs = strings(raw.input_shards, `${label}.input_shards`);- const assignedOutputs = strings(- raw.output_shards,- `${label}.output_shards`,- );- if (assignedInputs.length !== assignedOutputs.length)- throw new Error(- `${label} input_shards and output_shards lengths must match`,- );- if (!same(assignedOutputs, assignedInputs.map(outputShardName)))- throw new Error(`${label}.output_shards do not match its input_shards`);- });- const workers = payload.workers as Worker[];- const counts = new Map<string, number>();- for (const worker of workers)- for (const name of worker.input_shards)- counts.set(name, (counts.get(name) ?? 0) + 1);- const missing = missingNames(inputNames, counts.keys());- const duplicates = [...counts]- .filter(([, count]) => count > 1)- .map(([name]) => name)- .sort(compare);- const unexpected = missingNames(counts.keys(), inputNames);- if (missing.length || duplicates.length || unexpected.length)- throw new Error(- `${plan}: pool plan must assign each input shard exactly once; missing=${formatDiagnostic(missing)}; duplicates=${formatDiagnostic(duplicates)}; unexpected=${formatDiagnostic(unexpected)}`,- );- for (const [index, worker] of workers.entries()) {- const assigned = inputNames.filter(- (_, inputIndex) => inputIndex % workers.length === index,- );- if (!same(worker.input_shards, assigned))- throw new Error(- `${plan}: worker slot ${index + 1} does not match the deterministic round_robin assignment`,- );- }- return { inputs, outputNames, workers, bytes };-}--function validateWorker(- plan: string,- directory: string,- slotArgument: unknown,-): void {- const { workers, bytes } = validatePlan(plan, directory);- const slot = integer(slotArgument, "--slot", 1);- if (slot > workers.length)- throw new Error(`--slot must be at most ${workers.length}`);- const worker = workers[slot - 1]!;- let rows = 0;- const digest = createHash("sha256");- for (const [index, name] of worker.input_shards.entries()) {- const output = childPath(directory, worker.output_shards[index]!);- const [, outputs] = validateShard(childPath(directory, name), output);- const outputBytes = readFile(output);- rows += outputs.length;- digest- .update(worker.output_shards[index]!)- .update("\0")- .update(outputBytes)- .update("\0");- }- print(- "RANK_WORKER_RECEIPT " +- JSON.stringify({- output_shards: worker.output_shards.length,- outputs_sha256: digest.digest("hex"),- plan_sha256: createHash("sha256").update(bytes).digest("hex"),- ranking_worker_count: workers.length,- rows,- schema_version: 1,- slot: slot,- status: "complete",- }),- );-}--function validatePool(plan: string, directory: string): void {- const { inputs, outputNames, workers } = validatePlan(plan, directory);- const actual = shardNames(directory, "output");- const missing = missingNames(outputNames, actual);- const unexpected = missingNames(actual, outputNames);- if (missing.length || unexpected.length)- throw new Error(- `Rank pool outputs are incomplete; missing output shards=${formatDiagnostic(missing)}; unexpected output shards=${formatDiagnostic(unexpected)}`,- );- let rows = 0;- for (const input of inputs) {- const [, outputs] = validateShard(- input,- childPath(directory, outputShardName(basename(input))),- );- rows += outputs.length;- }- print(- `Validated ${workers.length} ranking workers, ${inputs.length} shards, and ${rows} ranking rows`,- );-}--export function rankPoolCommand(- command: Command,- args: string[],- posixHome = process.env.HOME,-): number {- const required =- command === "make-rank-pool-plan"- ? ["shard-dir", "usable-worker-slots", "out"]- : command === "validate-rank-worker"- ? ["plan", "shard-dir", "slot"]- : ["plan", "shard-dir"];- const integers =- command === "make-rank-pool-plan"- ? ["usable-worker-slots"]- : command === "validate-rank-worker"- ? ["slot"]- : [];- const syntax = required- .map((name) => `--${name} ${integers.includes(name) ? "INT" : "PATH"}`)- .join(" ");- const usage = `usage: launch_codex_security_mcp[.cmd] --helper ${command} [-h] ${syntax}`;- try {- const values = argumentsFor(args, required, integers);- if (values.help) {- const description =- command === "make-rank-pool-plan"- ? "Assign rank shards to a deterministic bounded worker pool. Usable worker slots are capped at 6."- : command === "validate-rank-worker"- ? "Validate one assigned ranking-worker slot and emit its completion receipt."- : "Validate a rank pool plan and every assigned shard output.";- print(- `${usage}\n\n${description}\n\noptions:\n -h, --help show this help message and exit\n${required.map((name) => ` --${name} ${integers.includes(name) ? "INT" : "PATH"}`).join("\n")}`,- );- return 0;- }- if (command === "make-rank-pool-plan") {- const slots = values["usable-worker-slots"] as number;- if (slots < 1)- throw new Error("--usable-worker-slots must be at least 1");- const directory = worklistPath(values["shard-dir"] as string, posixHome);- const output = worklistPath(values.out as string, posixHome);- makePlan(directory, slots, output);- } else {- const plan = worklistPath(values.plan as string, posixHome);- const directory = worklistPath(values["shard-dir"] as string, posixHome);- if (command === "validate-rank-worker")- validateWorker(plan, directory, values.slot);- else validatePool(plan, directory);- }- return 0;- } catch (error) {- return reportCommandError(error, command, usage);- }-}diff --git a/plugins/codex-security/mcp-app/src/helpers/rank-shards.ts b/plugins/codex-security/mcp-app/src/helpers/rank-shards.tsdeleted file mode 100644index d512cd93a..000000000--- a/plugins/codex-security/mcp-app/src/helpers/rank-shards.ts+++ /dev/null@@ -1,266 +0,0 @@-import { readdirSync, statSync } from "node:fs";-import { basename, sep, win32 } from "node:path";-import { windowsBinding } from "../native";-import {- pathText,- widePath,- windowsFileSystem,-} from "../../../native/windows-files.mjs";-import { formatDiagnostic } from "./json";-import { mkdir, normalizePath } from "./helper-files";-import { decodePosixBytes, encodePosixPath } from "./posix-path";-import {- argumentsFor,- loadRankRows,- missingNames,- print,- reportCommandError,- requireUniquePaths,- worklistPath,- writeRankRows,- type RankRow,-} from "./rank-worklists";--type Command =- "make-rank-shards" | "validate-rank-shard" | "merge-rank-outputs";--export function outputShardName(name: string): string {- return name.replaceAll(".input.jsonl", ".output.jsonl");-}--export function childPath(directory: string, name: string): string {- return normalizePath(- process.platform === "win32"- ? win32.join(- directory.length === 2 && directory[1] === ":"- ? `${directory}.`- : directory,- name,- )- : directory + sep + name,- );-}--export function shardNames(- directory: string,- kind: "input" | "output",-): string[] {- const matches = (name: string) => {- // Python's Unicode case-insensitive globbing includes these ASCII equivalents.- const matched =- process.platform === "win32"- ? name.replace(/[İı]/gu, "i").replace(/ſ/gu, "s").toLowerCase()- : name;- return (- matched.startsWith("rank-shard-") && matched.endsWith(`.${kind}.jsonl`)- );- };- try {- if (process.platform === "win32")- return windowsFileSystem(windowsBinding())- .entriesWithTypes(widePath(directory))- .map((entry) => pathText(entry.name))- .filter(matches);- return readdirSync(encodePosixPath(directory), { encoding: "buffer" })- .map(decodePosixBytes)- .filter(matches);- } catch (error) {- // pathlib glob ignores directory enumeration OSErrors.- if (!(error instanceof Error) || !("errno" in error || "winerror" in error))- throw error;- return [];- }-}--export function discoverInputShards(directory: string): string[] {- let isDirectory = false;- try {- isDirectory = (- process.platform === "win32"- ? windowsFileSystem(windowsBinding()).stat(widePath(directory))- : statSync(encodePosixPath(directory))- ).isDirectory();- } catch (error) {- const { code, winerror } = error as NodeJS.ErrnoException & {- winerror?: number;- };- if (- !["ENOENT", "ENOTDIR", "EBADF", "ELOOP"].includes(code ?? "") &&- winerror !== 21 &&- winerror !== 123- )- throw error;- }- if (!isDirectory)- throw new Error(`Rank shard directory missing: ${directory}`);- const names = shardNames(directory, "input");- const actual = new Set(names);- const expected = names.map(- (_, index) =>- `rank-shard-${String(index + 1).padStart(4, "0")}.input.jsonl`,- );- if (expected.some((name) => !actual.has(name)))- throw new Error(- `Rank input shards must use contiguous canonical names; expected=${formatDiagnostic(expected)}; actual=${formatDiagnostic(names)}`,- );- return expected.map((name) => childPath(directory, name));-}--export function validateShard(- input: string,- output: string,-): [RankRow[], RankRow[]] {- const inputs = loadRankRows(input, false, "Rank input shard");- requireUniquePaths(inputs, `Rank input shard ${basename(input)}`);- const outputs = loadRankRows(output, true, "Rank output shard");- requireUniquePaths(outputs, `Rank output shard ${basename(output)}`);- const expected = new Map(inputs.map((row) => [row.path, row.area]));- const actual = new Set(outputs.map((row) => row.path));- const missing = missingNames(expected.keys(), actual);- const unknown = missingNames(actual, expected.keys());- if (missing.length || unknown.length)- throw new Error(- `${output}: paths do not match its input shard; missing=${formatDiagnostic(missing)}; unknown=${formatDiagnostic(unknown)}`,- );- for (const row of outputs)- if (row.area !== expected.get(row.path))- throw new Error(- `${output}: area does not match rank input for ${row.path}`,- );- return [inputs, outputs];-}--function makeShards(- inputArgument: string,- directoryArgument: string,- maximum: number,- posixHome: string | undefined,-): void {- if (maximum < 1) throw new Error("--max-rows must be at least 1");- const input = worklistPath(inputArgument, posixHome);- const rows = loadRankRows(input, false);- requireUniquePaths(rows, "Rank input");- const directory = worklistPath(directoryArgument, posixHome);- mkdir(directory);- if (- shardNames(directory, "input").length ||- shardNames(directory, "output").length- )- throw new Error(- `Rank shard directory already contains shard files: ${directory}`,- );- let count = 0;- const size = maximum;- for (let start = 0; start < rows.length; start += size) {- const name = `rank-shard-${String(++count).padStart(4, "0")}.input.jsonl`;- writeRankRows(childPath(directory, name), rows.slice(start, start + size));- }- print(`Wrote ${count} rank shards to ${directory}`);-}--function mergeShards(- inputArgument: string,- directoryArgument: string,- outputArgument: string,- posixHome: string | undefined,-): void {- const input = worklistPath(inputArgument, posixHome);- const authoritative = loadRankRows(input, false);- requireUniquePaths(authoritative, "Rank input");- const directory = worklistPath(directoryArgument, posixHome);- const shards = discoverInputShards(directory);- const expected = shards.map((path) => outputShardName(basename(path)));- const actual = shardNames(directory, "output");- const missing = missingNames(expected, actual);- const unexpected = missingNames(actual, expected);- if (missing.length || unexpected.length) {- const details: string[] = [];- if (missing.length)- details.push(`missing output shards ${formatDiagnostic(missing)}`);- if (unexpected.length)- details.push(`unexpected output shards ${formatDiagnostic(unexpected)}`);- throw new Error(`Rank shard outputs are incomplete: ${details.join("; ")}`);- }- let inputIndex = 0;- let partitionMatches = true;- const outputByPath = new Map<string, RankRow>();- for (const shard of shards) {- const outputShard = childPath(directory, outputShardName(basename(shard)));- const [inputs, outputs] = validateShard(shard, outputShard);- for (const row of inputs) {- const expectedRow = authoritative[inputIndex++];- partitionMatches &&=- expectedRow !== undefined &&- row.path === expectedRow.path &&- row.area === expectedRow.area &&- row.preview === expectedRow.preview;- }- for (const row of outputs) {- if (outputByPath.has(row.path))- throw new Error(`Rank outputs contain duplicate path: ${row.path}`);- outputByPath.set(row.path, row);- }- }- if (inputIndex !== authoritative.length || !partitionMatches)- throw new Error(- "Rank input shards do not exactly partition the authoritative rank input",- );- const merged = authoritative.map((row) => outputByPath.get(row.path)!);- const output = worklistPath(outputArgument, posixHome);- writeRankRows(output, merged);- print(`Merged ${merged.length} ranking rows into ${output}`);-}--export function rankShardsCommand(- command: Command,- args: string[],- posixHome = process.env.HOME,-): number {- const required =- command === "make-rank-shards"- ? ["rank-input", "out-dir"]- : command === "validate-rank-shard"- ? ["input", "output"]- : ["rank-input", "shard-dir", "out"];- const integers = command === "make-rank-shards" ? ["max-rows"] : [];- const usage = `usage: launch_codex_security_mcp[.cmd] --helper ${command} [-h] ${required.map((name) => `--${name} PATH`).join(" ")}${integers.length ? " [--max-rows INT]" : ""}`;- try {- const values = argumentsFor(args, required, integers);- if (values.help) {- const description =- command === "make-rank-shards"- ? "Partition rank_input.jsonl into deterministic worker input shards."- : command === "validate-rank-shard"- ? "Validate one worker output against its rank input shard."- : "Validate worker shard outputs and create rank_output.jsonl.";- print(- `${usage}\n\n${description}\n\noptions:\n -h, --help show this help message and exit\n${required.map((name) => ` --${name} PATH`).join("\n")}${integers.length ? "\n --max-rows INT Maximum rows per shard. Defaults to 150." : ""}`,- );- return 0;- }- if (command === "make-rank-shards") {- const maximum = (values["max-rows"] ?? 150) as number;- makeShards(- values["rank-input"] as string,- values["out-dir"] as string,- maximum,- posixHome,- );- } else if (command === "validate-rank-shard") {- const input = worklistPath(values.input as string, posixHome);- const output = worklistPath(values.output as string, posixHome);- const [, rows] = validateShard(input, output);- print(`Validated ${rows.length} ranking rows in ${output}`);- } else {- mergeShards(- values["rank-input"] as string,- values["shard-dir"] as string,- values.out as string,- posixHome,- );- }- return 0;- } catch (error) {- return reportCommandError(error, command, usage);- }-}diff --git a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.tsindex 039d9b480..8e3c1a690 100644--- a/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts+++ b/plugins/codex-security/mcp-app/src/helpers/rank-worklists.ts@@ -15,12 +15,9 @@ export interface RankRow {path: string;area: string;preview?: string;- reason?: string;- score?: number;- include?: boolean;}const blank = (value: string) => value.trim().length === 0;-export function compare(left: string, right: string): number {+function compare(left: string, right: string): number {const a = Array.from(left, (character) => character.codePointAt(0)!);const b = Array.from(right, (character) => character.codePointAt(0)!);for (let index = 0; index < Math.min(a.length, b.length); index++) {@@ -29,7 +26,7 @@ export function compare(left: string, right: string): number {return a.length - b.length;}-export function missingNames(+function missingNames(expected: Iterable<string>,actual: Iterable<string>,): string[] {@@ -39,15 +36,9 @@ export function missingNames(.sort(compare);}-export function loadRankRows(- path: string,- selection: boolean,- label = selection ? "Rank output" : "Rank input",-): RankRow[] {- const fields = selection- ? ["path", "area", "score", "include", "reason"]- : ["path", "area", "preview"];- const rows = Array.from(readUtf8Lines(path, label), (line, index) => {+export function loadRankRows(path: string): RankRow[] {+ const fields = ["path", "area", "preview"];+ const rows = Array.from(readUtf8Lines(path, "Rank input"), (line, index) => {const fail = (message: string): never => {throw new Error(`${path}:${index + 1}: ${message}`);};@@ -69,9 +60,7 @@ export function loadRankRows(if (unexpected.length)details.push(`unexpected fields ${formatDiagnostic(unexpected)}`);if (details.length) fail(details.join("; "));- for (const field of selection- ? ["path", "area"]- : ["path", "area", "preview"]) {+ for (const field of fields) {if (typeof row[field] !== "string" ||(field === "path" && blank(row[field]))@@ -80,15 +69,6 @@ export function loadRankRows(`${field} must be ${field === "path" ? "a non-empty string" : "a string"}`,);}- if (selection) {- if (typeof row.score !== "number" || !Number.isInteger(row.score))- fail("score must be an integer from 1 through 10");- if ((row.score as number) < 1 || (row.score as number) > 10)- fail("score must be from 1 through 10");- if (typeof row.include !== "boolean") fail("include must be a boolean");- if (typeof row.reason !== "string" || blank(row.reason))- fail("reason must be a non-empty string");- }return row as unknown as RankRow;});return rows;@@ -122,18 +102,14 @@ export class ArgumentError extends Error {}export function argumentsFor(args: string[],required: readonly string[],- integerOptions: readonly string[] = [],-): Record<string, string | number | true> {+): Record<string, string | true> {try {const { values }: { values: Record<string, string | boolean | undefined> } =parseArgs({args,options: {...Object.fromEntries(- [...required, ...integerOptions].map((name) => [- name,- { type: "string" as const },- ]),+ required.map((name) => [name, { type: "string" as const }]),),help: { type: "boolean", short: "h" },},@@ -144,16 +120,10 @@ export function argumentsFor(throw new Error(`Missing required options: ${missing.map((name) => `--${name}`).join(", ")}`,);- const result: Record<string, string | number | true> = {};+ const result: Record<string, string | true> = {};for (const [name, value] of Object.entries(values)) {if (typeof value !== "string") continue;- if (!integerOptions.includes(name)) result[name] = value;- else {- const number = Number(value);- if (!/^[+-]?\d+$/u.test(value.trim()) || !Number.isInteger(number))- throw new Error(`--${name} must be an integer`);- result[name] = number;- }+ result[name] = value;}return result;} catch (error) {diff --git a/plugins/codex-security/native/README.md b/plugins/codex-security/native/README.mdindex 3eeb022ca..e8f21d21c 100644--- a/plugins/codex-security/native/README.md+++ b/plugins/codex-security/native/README.md@@ -33,7 +33,7 @@ Windows uses `windows-binding.mts` and the same Rust crate. `WindowsHandle` ownsThe binding exposes synchronous file and recursive directory creation, attributes and reparse tags, identity and final/opened names, read/write, and exact-handle rename and deletion. Rust's `File` supplies ordinary I/O. Calls return numeric Windows errors, including 6 for closed handles. Buffer ranges and path encoding are checked before use. Overlapped handles are unsupported because pending operations could retain native buffers beyond the call. Path authorization, ancestor traversal, and reparse-point policy remain the caller's responsibility.-Four additional operations preserve Windows strings at the Node boundary. `windowsArguments` returns the complete OS argument vector, including the executable and Node options, using Rust's CRT-compatible parser. `windowsEnvironment` reads one wide environment name and distinguishes an absent value (`null`) from an empty buffer. `windowsAbsolutePath` resolves against the native current directory and drive directories without requiring the destination to exist. `windowsDirectoryEntries` uses `std::fs::read_dir` and cached `DirEntry::file_type()` values without opening each child; names remain UTF-16LE, and construction or iteration failures return their numeric Windows error and an empty array. Directory symlinks and junctions have both directory and symbolic-link flags. The typed adapter exposes this enumerator through `entriesWithTypes`, which `resolve-security-md --list` uses on Windows. Assessment validation, deep-review worklists, and rank shard/pool helpers share the typed wide-path reader and writer.+Four additional operations preserve Windows strings at the Node boundary. `windowsArguments` returns the complete OS argument vector, including the executable and Node options, using Rust's CRT-compatible parser. `windowsEnvironment` reads one wide environment name and distinguishes an absent value (`null`) from an empty buffer. `windowsAbsolutePath` resolves against the native current directory and drive directories without requiring the destination to exist. `windowsDirectoryEntries` uses `std::fs::read_dir` and cached `DirEntry::file_type()` values without opening each child; names remain UTF-16LE, and construction or iteration failures return their numeric Windows error and an empty array. Directory symlinks and junctions have both directory and symbolic-link flags. The typed adapter exposes this enumerator through `entriesWithTypes`, which `resolve-security-md --list` uses on Windows. Assessment validation, and deep-review worklists share the typed wide-path reader and writer.`windows-files.mts` uses native absolute and final paths, preserving raw UTF-16 and the returned device prefix. Non-strict `realpath` resolves the nearest existing ancestor of a missing path; dangling links and other access errors remain errors. Callers must check containment independently. It follows native Windows path behavior rather than emulating Python's special cases for device names, whitespace, verbatim dot segments, and prefix removal. `stat(path, false)` retains exact symbolic-link and reparse-point metadata so callers can reject junction traversal independently of the enumerator's link label. The SDK's public runtime floor remains Node 22.13.0. Node 20.0.0 is an additional native-foundation compatibility proof; it does not change the SDK engine requirement.diff --git a/plugins/codex-security/native/examples/windows-wide-launcher.rs b/plugins/codex-security/native/examples/windows-wide-launcher.rsindex 63c05145e..65be54713 100644--- a/plugins/codex-security/native/examples/windows-wide-launcher.rs+++ b/plugins/codex-security/native/examples/windows-wide-launcher.rs@@ -409,35 +409,30 @@ fn main() -> std::io::Result<()> {.join(&replacement_output);fs::create_dir_all(worklist_sentinel.parent().unwrap())?;fs::write(&worklist_sentinel, "worklist output sentinel")?;- for (command, input_flag, row) in [- (+ fs::write(+ repo.join(&input_name),+ "{\"path\":\"source.py\",\"area\":\"src\",\"preview\":\"source line\"}\n",+ )?;+ for prefix in [repo.clone(), PathBuf::from("~"), PathBuf::from(".")] {+ let child = invoke("copy-deep-review-input",- "--rank-input",- "{\"path\":\"source.py\",\"area\":\"src\",\"preview\":\"source line\"}\n",- ),- (- "select-deep-review-input",- "--rank-output",- "{\"path\":\"source.py\",\"area\":\"src\",\"score\":5,\"include\":true,\"reason\":\"source line\"}\n",- ),- ] {- fs::write(repo.join(&input_name), row)?;- for prefix in [repo.clone(), PathBuf::from("~"), PathBuf::from(".")] {- let child = invoke(command, &[- input_flag.into(), prefix.join(&input_name),- "--out".into(), prefix.join(&worklist_dir).join(&output_name),- ])?;- if !child.status.success()- || !child.stderr.is_empty()- || fs::read(&worklist_output)? != b"{\"path\":\"source.py\",\"area\":\"src\"}\r\n"- {- return Err(io::Error::other(format!(- "Wide deep-review helper failed: {}",- String::from_utf8_lossy(&child.stderr)- )));- }- fs::remove_file(&worklist_output)?;+ &[+ "--rank-input".into(),+ prefix.join(&input_name),+ "--out".into(),+ prefix.join(&worklist_dir).join(&output_name),+ ],+ )?;+ if !child.status.success()+ || !child.stderr.is_empty()+ || fs::read(&worklist_output)? != b"{\"path\":\"source.py\",\"area\":\"src\"}\r\n"+ {+ return Err(io::Error::other(format!(+ "Wide deep-review helper failed: {}",+ String::from_utf8_lossy(&child.stderr)+ )));}+ fs::remove_file(&worklist_output)?;}if fs::read(&worklist_sentinel)? != b"worklist output sentinel"|| fs::read(repo.join(raw("input-", 0xfffd)))? != b"invalid replacement input"@@ -446,165 +441,6 @@ fn main() -> std::io::Result<()> {"Deep-review helper changed a replacement path",));}- let shards_name = raw("shards-", 0xdc80);- let shards = repo.join(&shards_name);- let replacement_shards = repo.join(raw("shards-", 0xfffd));- fs::create_dir(&replacement_shards)?;- let shard_sentinel = replacement_shards.join("rank-shard-0001.input.jsonl");- fs::write(&shard_sentinel, "replacement shard sentinel")?;- let merge_parent = raw("merged-", 0xd800);- let merged = repo.join(&merge_parent).join(&output_name);- let merge_sentinel = repo.join(raw("merged-", 0xfffd)).join(&replacement_output);- fs::create_dir_all(merge_sentinel.parent().unwrap())?;- fs::write(&merge_sentinel, "replacement merge sentinel")?;- let input_rows = [- "{\"path\":\"first.py\",\"area\":\"src\",\"preview\":\"first\"}\r\n",- "{\"path\":\"second.py\",\"area\":\"src\",\"preview\":\"second\"}\r\n",- ];- let output_rows = [- "{\"path\":\"first.py\",\"area\":\"src\",\"score\":5,\"include\":true,\"reason\":\"review\"}\r\n",- "{\"path\":\"second.py\",\"area\":\"src\",\"score\":6,\"include\":true,\"reason\":\"review\"}\r\n",- ];- fs::write(repo.join(&input_name), input_rows.concat())?;- let made = invoke(- "make-rank-shards",- &[- "--rank-input".into(),- repo.join(&input_name),- "--out-dir".into(),- Path::new("~").join(&shards_name),- "--max-rows".into(),- "1".into(),- ],- )?;- if !made.status.success() || !made.stderr.is_empty() {- return Err(io::Error::other(format!(- "Wide shard creation failed: {}",- String::from_utf8_lossy(&made.stderr)- )));- }- for (index, input_row) in input_rows.iter().enumerate() {- let stem = format!("rank-shard-{:04}", index + 1);- if fs::read(shards.join(format!("{stem}.input.jsonl")))? != input_row.as_bytes() {- return Err(io::Error::other("Wide input shard bytes changed"));- }- fs::write(- shards.join(format!("{stem}.output.jsonl")),- output_rows[index],- )?;- }- let validated = invoke(- "validate-rank-shard",- &[- "--input".into(),- Path::new(&shards_name).join("rank-shard-0001.input.jsonl"),- "--output".into(),- Path::new(&shards_name).join("rank-shard-0001.output.jsonl"),- ],- )?;- let merge_args = [- "--rank-input".into(),- Path::new(".").join(&input_name),- "--shard-dir".into(),- shards.clone(),- "--out".into(),- Path::new("~").join(&merge_parent).join(&output_name),- ];- let merge_result = invoke("merge-rank-outputs", &merge_args)?;- if !validated.status.success()- || !validated.stderr.is_empty()- || !merge_result.status.success()- || !merge_result.stderr.is_empty()- || fs::read(&merged)? != output_rows.concat().as_bytes()- {- return Err(io::Error::other(format!(- "Wide shard validation/merge failed: {}{}",- String::from_utf8_lossy(&validated.stderr),- String::from_utf8_lossy(&merge_result.stderr)- )));- }- let mut malformed_name = raw("rank-shard-", 0xdfff);- malformed_name.push(".input.jsonl");- fs::write(shards.join(&malformed_name), "")?;- let malformed = invoke("merge-rank-outputs", &merge_args)?;- if malformed.status.code() != Some(1)- || !String::from_utf8_lossy(&malformed.stderr)- .contains("Rank input shards must use contiguous canonical names")- || !String::from_utf8_lossy(&malformed.stderr)- .contains("rank-shard-\\udfff.input.jsonl")- || fs::read(&merged)? != output_rows.concat().as_bytes()- || fs::read(&shard_sentinel)? != b"replacement shard sentinel"- || fs::read(&merge_sentinel)? != b"replacement merge sentinel"- || fs::read(repo.join(raw("input-", 0xfffd)))? != b"invalid replacement input"- {- return Err(io::Error::other(- "Wide shard discovery or replacement paths changed",- ));- }- fs::remove_file(shards.join(&malformed_name))?;- let pool_name = raw("pool-", 0xd800);- let plan_name = raw("plan-", 0xdfff);- let pool = repo.join(&pool_name);- let pool_shards = pool.join("rank_shards");- fs::create_dir(&pool)?;- fs::rename(&shards, &pool_shards)?;- let plan = pool.join(&plan_name);- let plan_sentinel = pool.join(raw("plan-", 0xfffd));- fs::write(&plan_sentinel, "replacement plan sentinel")?;- let made_plan = invoke(- "make-rank-pool-plan",- &[- "--shard-dir".into(),- Path::new("~").join(&pool_name).join("rank_shards"),- "--usable-worker-slots".into(),- "2".into(),- "--out".into(),- Path::new(&pool_name).join(&plan_name),- ],- )?;- if !made_plan.status.success() || !made_plan.stderr.is_empty() {- return Err(io::Error::other(format!(- "Wide pool plan creation failed: {}",- String::from_utf8_lossy(&made_plan.stderr)- )));- }- let plan_bytes = fs::read(&plan)?;- let expected_plan = concat!(- "{\r\n \"ranking_worker_count\": 2,\r\n \"schema_version\": 1,\r\n",- " \"shard_count\": 2,\r\n \"strategy\": \"round_robin\",\r\n \"workers\": [\r\n",- " {\r\n \"input_shards\": [\r\n \"rank-shard-0001.input.jsonl\"\r\n ],\r\n",- " \"output_shards\": [\r\n \"rank-shard-0001.output.jsonl\"\r\n ],\r\n \"slot\": 1\r\n },\r\n",- " {\r\n \"input_shards\": [\r\n \"rank-shard-0002.input.jsonl\"\r\n ],\r\n",- " \"output_shards\": [\r\n \"rank-shard-0002.output.jsonl\"\r\n ],\r\n \"slot\": 2\r\n }\r\n ]\r\n}\r\n",- );- if plan_bytes != expected_plan.as_bytes() {- return Err(io::Error::other("Wide pool assignment bytes changed"));- }- let pool_args = [- "--plan".into(),- plan.clone(),- "--shard-dir".into(),- Path::new(".").join(&pool_name).join("rank_shards"),- ];- let mut worker_args = pool_args.to_vec();- worker_args.extend(["--slot".into(), "1".into()]);- let worker = invoke("validate-rank-worker", &worker_args)?;- let complete = invoke("validate-rank-pool", &pool_args)?;- if !worker.status.success()- || !worker.stderr.is_empty()- || !worker.stdout.starts_with(b"RANK_WORKER_RECEIPT ")- || !complete.status.success()- || !complete.stderr.is_empty()- || complete.stdout != b"Validated 2 ranking workers, 2 shards, and 2 ranking rows\r\n"- || fs::read(&plan)? != plan_bytes- || fs::read(&plan_sentinel)? != b"replacement plan sentinel"- {- return Err(io::Error::other(format!(- "Wide pool validation failed: {}{}",- String::from_utf8_lossy(&worker.stderr),- String::from_utf8_lossy(&complete.stderr)- )));- }let scopes_name = raw("requested-", 0xd800);let manifest_name = raw("manifest-", 0xdc80);let coverage_name = raw("coverage-", 0xdfff);diff --git a/plugins/codex-security/scripts/generate_rank_input.py b/plugins/codex-security/scripts/generate_rank_input.pyindex dd02ec291..754abc9d1 100644--- a/plugins/codex-security/scripts/generate_rank_input.py+++ b/plugins/codex-security/scripts/generate_rank_input.py@@ -3,14 +3,13 @@This script stays deliberately model-free:-- `make-repo-rank-input` creates the deterministic repository or scoped-path- JSONL candidate worklist that ranking subagents consume.+- `make-repo-scope-input` lists every explicitly scoped repository file.- `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidateworklist from Git changed paths. It supports committed revision diffs andlocal working-tree patches.Candidate selection uses repository scope, native tool ignore rules, and binary detection,-not filename or directory classifications. Ranking inputs use bounded source previews.+not filename or directory classifications. Diff inputs use bounded source previews."""from __future__ import annotations@@ -41,29 +40,6 @@ def parse_args() -> argparse.Namespace:parser = argparse.ArgumentParser(description="Codex Security scan worklist helper.")subparsers = parser.add_subparsers(dest="command", required=True)- make = subparsers.add_parser(- "make-repo-rank-input",- help="Create rank_input.jsonl for subagent-based file ranking.",- )- make.add_argument("--repo", required=True, help="Repository root.")- make.add_argument(- "--scope",- default=".",- help="Path within the repository to scan. Defaults to the repository root.",- )- make.add_argument(- "--scopes-file",- help="JSON array of repository-relative files and directories to scan together.",- )- make.add_argument("--out", required=True, help="Output rank_input.jsonl path.")- make.add_argument("--area", default="", help="Area label. Defaults to scope.")- make.add_argument(- "--preview-bytes",- type=int,- default=DEFAULT_PREVIEW_BYTES,- help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",- )-scoped = subparsers.add_parser("make-repo-scope-input",help="List every explicitly scoped file without ranking or reading its contents.",@@ -90,7 +66,7 @@ def parse_args() -> argparse.Namespace:)diff.add_argument("--head", default="HEAD", help="Git diff head revision.")diff.add_argument("--out", required=True, help="Output rank_input.jsonl path.")- diff.add_argument("--area", default="diff", help="Area label for ranking rows.")+ diff.add_argument("--area", default="diff", help="Area label for changed-file rows.")diff.add_argument("--preview-bytes",type=int,@@ -101,39 +77,32 @@ def parse_args() -> argparse.Namespace:return parser.parse_args()-def resolve_scope(- repo: Path,- scope: str,- *,- expand_user: bool = True,- reject_symlinks: bool = False,-) -> Path:- scope_path = Path(scope).expanduser() if expand_user else Path(scope)+def resolve_scope(repo: Path, scope: str) -> Path:+ scope_path = Path(scope)stream = windows_stream_component(scope_path)if stream is not None:raise SystemExit(f"Scope must not use an NTFS alternate data stream: {stream}")if not scope_path.is_absolute():scope_path = repo / scope_path- if reject_symlinks:- repository = repo.resolve()+ repository = repo.resolve()+ try:+ relative = scope_path.relative_to(repository)+ except ValueError as exc:+ raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc+ ancestor = repository+ for part in relative.parts:+ if part == "..":+ if ancestor == repository:+ raise SystemExit(f"Scope must be inside repo: {scope_path}")+ ancestor = ancestor.parent+ continue+ ancestor /= parttry:- relative = scope_path.relative_to(repository)- except ValueError as exc:- raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc- ancestor = repository- for part in relative.parts:- if part == "..":- if ancestor == repository:- raise SystemExit(f"Scope must be inside repo: {scope_path}")- ancestor = ancestor.parent- continue- ancestor /= part- try:- metadata = ancestor.stat(follow_symlinks=False)- except OSError as exc:- raise SystemExit(f"Scope path not found: {ancestor}") from exc- if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:- raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")+ metadata = ancestor.stat(follow_symlinks=False)+ except OSError as exc:+ raise SystemExit(f"Scope path not found: {ancestor}") from exc+ if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:+ raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")scope_path = scope_path.resolve()repo_resolved = repo.resolve()try:@@ -221,54 +190,6 @@ def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)-def make_repo_rank_input(args: argparse.Namespace) -> None:- repo = Path(args.repo).expanduser().resolve()- if not repo.is_dir():- raise SystemExit(f"Repo path not found: {repo}")- scopes = [args.scope]- explicit_scopes = args.scopes_file is not None- if explicit_scopes:- scopes = load_scopes_file(Path(args.scopes_file).expanduser())-- resolved_scopes = [- resolve_scope(repo, scope, expand_user=not explicit_scopes) for scope in scopes- ]- directly_requested_files = {- scope_abs for scope_abs in resolved_scopes if explicit_scopes and scope_abs.is_file()- }- rows_by_path: dict[str, JsonRow] = {}- for scope_abs in resolved_scopes:- scope_rel = scope_abs.relative_to(repo)- area = args.area or scope_rel.as_posix()- candidates = scope_candidates(repo, scope_abs)- for path in candidates:- try:- if path.is_symlink() or not path.is_file():- continue- path.resolve(strict=True).relative_to(repo)- except (OSError, ValueError):- continue- rel = path.relative_to(repo)- directly_requested = path in directly_requested_files- if ".git" in rel.parts:- continue-- preview, is_binary = preview_for(path, args.preview_bytes)- if is_binary or is_binary_file(path):- if not directly_requested:- continue- preview = ""- rows_by_path.setdefault(- rel.as_posix(),- {"path": rel.as_posix(), "area": area, "preview": preview},- )-- rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"]))- output = Path(args.out).expanduser()- write_jsonl(output, rows)- print(f"Wrote {len(rows)} rows to {output}")--def make_repo_scope_input(args: argparse.Namespace) -> None:repo = Path(args.repo).expanduser().resolve()if not repo.is_dir():@@ -277,7 +198,7 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:scopes = load_scopes_file(Path(args.scopes_file).expanduser())rows_by_path: dict[str, JsonRow] = {}for scope in scopes:- scope_path = resolve_scope(repo, scope, expand_user=False, reject_symlinks=True)+ scope_path = resolve_scope(repo, scope)candidates = scope_candidates(repo, scope_path)for path in candidates:try:@@ -410,9 +331,7 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:def main() -> None:args = parse_args()- if args.command == "make-repo-rank-input":- make_repo_rank_input(args)- elif args.command == "make-repo-scope-input":+ if args.command == "make-repo-scope-input":make_repo_scope_input(args)elif args.command == "make-diff-rank-input":make_diff_rank_input(args)diff --git a/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md b/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.mdindex 4b9a581f9..e7ccde1fe 100644--- a/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md+++ b/plugins/codex-security/skills/security-scan/references/scan-artifacts-and-ledger.md@@ -55,6 +55,8 @@ The parent agent must reconcile validation and attack-path subagent outputs befo## Scoped Deep Review+The legacy ranking helpers have been removed: `make-repo-rank-input`, `select-deep-review-input`, `make-rank-shards`, `validate-rank-shard`, `merge-rank-outputs`, `make-rank-pool-plan`, `validate-rank-worker`, and `validate-rank-pool`. Use `make-repo-scope-input --repo <repo_root> --scopes-file <scopes_json> --out <output_jsonl>` for explicit repository scopes. Diff scans retain `make-diff-rank-input` and `copy-deep-review-input` as shown below, including their existing arguments and worklist formats.+- Use `deep_review_input.jsonl` as the canonical changed-file review worklist for diff scans.- For diff-scoped scans, generate `rank_input.jsonl` deterministically from changed text files regardless of filename, extension, or directory, with `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode revisions --head <head> --out <discovery_dir>/rank_input.jsonl` for PR, commit, and branch diffs, or `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode local-patch --out <discovery_dir>/rank_input.jsonl` for a local patch, then copy every row into `deep_review_input.jsonl` with `<plugin_dir>/scripts/launch_codex_security_mcp --helper copy-deep-review-input --rank-input <discovery_dir>/rank_input.jsonl --out <discovery_dir>/deep_review_input.jsonl`. On Windows, use the [PowerShell copy command](#windows-worklist-copy) below.- Diff-scoped scans do not rank or drop changed files before deep review. Every row in diff `rank_input.jsonl` must be copied into `deep_review_input.jsonl` and receive a full-file review receipt.diff --git a/plugins/codex-security/tests/LAYER1_FOLLOWUPS.md b/plugins/codex-security/tests/LAYER1_FOLLOWUPS.mdindex 72905b2d1..322834e07 100644--- a/plugins/codex-security/tests/LAYER1_FOLLOWUPS.md+++ b/plugins/codex-security/tests/LAYER1_FOLLOWUPS.md@@ -4,13 +4,12 @@ This file is the source of truth for portable deterministic, no-model Codex Secu## Covered-- **Python utilities:** unit and CLI tests cover capability preflight, scan contract finalization, rank-input generation and selection, and report format validation.+- **Python utilities:** unit and CLI tests cover capability preflight, scan contract finalization, diff and scoped-file worklist generation, and report format validation.- **Schema validation:** Draft 2020-12 validators and checked-in completed-scan examples are validated with `jsonschema`, including RFC 3339 formats.- **Plugin metadata:** the plugin manifest, referenced assets, skill frontmatter, and agent interface YAML are parsed and checked for required fields and file consistency.- **Prompt linting:** agent default prompts are required to be non-empty.- **CLI and harness dry runs:** every Python script supports `--help`; major command paths run against temporary repositories and scan bundles without a model call.- **Static checks:** Ruff checks the plugin scripts and tests; the `--help` smoke test exercises top-level Python script imports.-- **Golden files:** deterministic repository ranking JSONL output is compared against a checked-in golden file.- **MCP dependencies and templates:** CI installs the MCP package with its frozen lockfile; its deterministic tests exercise Deep Scan prompt-template rendering.diff --git a/plugins/codex-security/tests/goldens/rank_input.jsonl b/plugins/codex-security/tests/goldens/rank_input.jsonldeleted file mode 100644index bb00ad8e6..000000000--- a/plugins/codex-security/tests/goldens/rank_input.jsonl+++ /dev/null@@ -1,2 +0,0 @@-{"path":"src/alpha.py","area":"src","preview":"alpha = 1"}-{"path":"src/zeta.py","area":"src","preview":"zeta = 2"}diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.pyindex afc86acfb..b0b741c06 100644--- a/plugins/codex-security/tests/test_generate_rank_input.py+++ b/plugins/codex-security/tests/test_generate_rank_input.py@@ -12,7 +12,6 @@ from source_cases import SOURCE_CASESPLUGIN_ROOT = Path(__file__).resolve().parents[1]SCRIPT = PLUGIN_ROOT / "scripts" / "generate_rank_input.py"-GOLDEN_DIR = Path(__file__).resolve().parent / "goldens"def run_repo_cli(@@ -58,27 +57,7 @@ def initialize_repo(repo: Path) -> None:git(repo, "config", "user.name", "Codex Security Tests")-def test_make_repo_rank_input_matches_golden_and_filters_noise(tmp_path: Path) -> None:- repo = tmp_path / "repo"- (repo / "src").mkdir(parents=True)- (repo / "tests").mkdir()- (repo / "src" / "zeta.py").write_text("zeta = 2", encoding="utf-8")- (repo / "src" / "alpha.py").write_text("alpha = 1", encoding="utf-8")- (repo / "src" / "binary.py").write_bytes(b"value\x00binary")- (repo / "src" / "binary.unlisted").write_bytes(b"x" * (64 * 1024 + 1) + b"\0binary")- (repo / "tests" / "ignored.py").write_text("ignored = True", encoding="utf-8")- (repo / "README.md").write_text("Deployment instructions", encoding="utf-8")- output = tmp_path / "rank_input.jsonl"-- run_repo_cli("make-repo-rank-input", repo, output, "--scope", "src")-- assert output.read_text(encoding="utf-8") == (GOLDEN_DIR / "rank_input.jsonl").read_text(- encoding="utf-8"- )--[email protected]("mode", ["repo", "revisions"])-def test_rank_input_keeps_large_generated_python(tmp_path: Path, mode: str) -> None:+def test_rank_input_keeps_large_generated_python(tmp_path: Path) -> None:repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)initialize_repo(repo)@@ -89,14 +68,10 @@ def test_rank_input_keeps_large_generated_python(tmp_path: Path, mode: str) -> N(repo / "src" / "normal.py").write_text("value = 1\n", encoding="utf-8")output = tmp_path / "rank_input.jsonl"- if mode == "repo":- command = "make-repo-rank-input"- arguments = ["--scope", "src"]- else:- git(repo, "add", ".")- git(repo, "commit", "-qm", "change")- command = "make-diff-rank-input"- arguments = ["--base", base, "--head", "HEAD"]+ git(repo, "add", ".")+ git(repo, "commit", "-qm", "change")+ command = "make-diff-rank-input"+ arguments = ["--base", base, "--head", "HEAD"]run_repo_cli(command, repo, output, *arguments, "--preview-bytes", "128")rows = read_jsonl(output)@@ -146,9 +121,8 @@ def test_revision_rank_input_classifies_bytes_beyond_the_preview_sample(tmp_pathassert read_jsonl(output) == [][email protected]("scope", [".", "src", "src/large.py", "explicit", "overlap", "diff"])def test_rank_input_streams_classification_and_bounds_previews(- tmp_path: Path, monkeypatch: pytest.MonkeyPatch, scope: str+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)@@ -159,20 +133,19 @@ def test_rank_input_streams_classification_and_bounds_previews(+ b"def outside():\n pass\n")output = tmp_path / "rank_input.jsonl"- arguments = ["make-repo-rank-input", "--repo", str(repo), "--out", str(output)]- if scope in {"explicit", "overlap"}:- scopes = tmp_path / "scopes.json"- scopes.write_text(- json.dumps(["src/large.py"] if scope == "explicit" else ["src", "src/large.py"])- )- arguments.extend(["--scopes-file", str(scopes)])- elif scope == "diff":- initialize_repo(repo)- git(repo, "commit", "--allow-empty", "-qm", "initial")- arguments[0] = "make-diff-rank-input"- arguments.extend(["--mode", "local-patch", "--base", "HEAD"])- else:- arguments.extend(["--scope", scope])+ initialize_repo(repo)+ git(repo, "commit", "--allow-empty", "-qm", "initial")+ arguments = [+ "make-diff-rank-input",+ "--repo",+ str(repo),+ "--out",+ str(output),+ "--mode",+ "local-patch",+ "--base",+ "HEAD",+ ]original_open = Path.openreads: list[int] = []@@ -207,7 +180,7 @@ def test_rank_input_streams_classification_and_bounds_previews(assert "outside" not in preview[email protected]("mode", ["repo", "revisions", "staged", "unstaged"])[email protected]("mode", ["revisions", "staged", "unstaged"])def test_rank_input_includes_source_cases(tmp_path: Path, mode: str) -> None:repo = tmp_path / "repo"repo.mkdir()@@ -225,50 +198,27 @@ def test_rank_input_includes_source_cases(tmp_path: Path, mode: str) -> None:if mode in {"revisions", "staged"}:git(repo, "add", ".")- if mode == "repo":- command = "make-repo-rank-input"- arguments = ["--scope", "."]- else:- diff_mode = "revisions" if mode == "revisions" else "local-patch"- command = "make-diff-rank-input"- arguments = ["--base", base, "--mode", diff_mode]- if mode == "revisions":- git(repo, "commit", "-qm", "change")- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])- git(repo, "checkout", "-q", base)+ diff_mode = "revisions" if mode == "revisions" else "local-patch"+ command = "make-diff-rank-input"+ arguments = ["--base", base, "--mode", diff_mode]+ if mode == "revisions":+ git(repo, "commit", "-qm", "change")+ arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])+ git(repo, "checkout", "-q", base)run_repo_cli(command, repo, output, *arguments)assert read_jsonl(output) == [{"path": case.path,- "area": "." if mode == "repo" else "diff",+ "area": "diff","preview": case.after,}for case in sorted(SOURCE_CASES, key=lambda case: case.path)]-def test_make_repo_rank_input_rejects_scope_outside_repo(tmp_path: Path) -> None:- repo = tmp_path / "repo"- repo.mkdir()- outside = tmp_path / "outside"- outside.mkdir()-- result = run_repo_cli(- "make-repo-rank-input",- repo,- tmp_path / "rank.jsonl",- "--scope",- str(outside),- check=False,- )-- assert result.returncode != 0- assert "Scope must be inside repo" in result.stderr---def test_make_repo_rank_input_does_not_follow_file_symlinks(tmp_path: Path) -> None:+def test_make_repo_scope_input_does_not_follow_file_symlinks(tmp_path: Path) -> None:repo = tmp_path / "repo"source = repo / "src"source.mkdir(parents=True)@@ -279,31 +229,15 @@ def test_make_repo_rank_input_does_not_follow_file_symlinks(tmp_path: Path) -> N(source / "inside-link.py").symlink_to(source / "runtime.py")output = tmp_path / "rank_input.jsonl"- run_repo_cli("make-repo-rank-input", repo, output, "--scope", "src")+ scopes = tmp_path / "scopes.json"+ scopes.write_text(json.dumps(["src"]))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))- assert read_jsonl(output) == [- {"path": "src/runtime.py", "area": "src", "preview": "runtime = True"}- ]+ assert read_jsonl(output) == [{"path": "src/runtime.py"}]assert "outside_secret" not in output.read_text(encoding="utf-8")-def test_make_repo_rank_input_preserves_legacy_tilde_scope(- tmp_path: Path, monkeypatch: pytest.MonkeyPatch-) -> None:- home = tmp_path / "home"- repo = home / "repo"- (repo / "src").mkdir(parents=True)- (repo / "src" / "runtime.py").write_text("runtime = True", encoding="utf-8")- monkeypatch.setenv("HOME", str(home))- monkeypatch.setenv("USERPROFILE", str(home))- output = tmp_path / "rank_input.jsonl"-- run_repo_cli("make-repo-rank-input", "~/repo", output, "--scope", "~/repo/src")-- assert [row["path"] for row in read_jsonl(output)] == ["src/runtime.py"]---def test_make_repo_rank_input_combines_explicit_files_and_directories(tmp_path: Path) -> None:+def test_make_repo_scope_input_combines_explicit_files_and_directories(tmp_path: Path) -> None:repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)(repo / "tests").mkdir()@@ -335,7 +269,7 @@ def test_make_repo_rank_input_combines_explicit_files_and_directories(tmp_path:)output = tmp_path / "rank_input.jsonl"- run_repo_cli("make-repo-rank-input", repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))assert [row["path"] for row in read_jsonl(output)] == sorted([@@ -352,8 +286,7 @@ def test_make_repo_rank_input_combines_explicit_files_and_directories(tmp_path:)[email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])-def test_repo_input_preserves_every_requested_directory_file(tmp_path: Path, command: str) -> None:+def test_repo_input_preserves_every_requested_directory_file(tmp_path: Path) -> None:repo = tmp_path / "repo"(repo / "src" / "tests").mkdir(parents=True)(repo / "src" / "examples").mkdir()@@ -369,7 +302,7 @@ def test_repo_input_preserves_every_requested_directory_file(tmp_path: Path, comscopes.write_text(json.dumps(["src", "src/runtime.py", "src/.git/config"]), encoding="utf-8")output = tmp_path / "scoped-source-input.jsonl"- run_repo_cli(command, repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))assert [row["path"] for row in read_jsonl(output)] == ["src/Dockerfile",@@ -429,11 +362,10 @@ def test_make_repo_scope_input_rejects_explicit_symlink_scopes(tmp_path: Path, sassert not output.exists()[email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])@pytest.mark.parametrize("scope", ["src", "."])@pytest.mark.parametrize("replacement", ["link", "loop", "file"])def test_repo_input_handles_replaced_tracked_directories(- tmp_path: Path, command: str, scope: str, replacement: str+ tmp_path: Path, scope: str, replacement: str) -> None:repo = tmp_path / "repo"nested = repo / "src" / "nested"@@ -459,7 +391,7 @@ def test_repo_input_handles_replaced_tracked_directories(scopes.write_text(json.dumps([scope]), encoding="utf-8")output = tmp_path / "input.jsonl"- run_repo_cli(command, repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))expected = ["src/handler.py"]if replacement == "file":@@ -469,10 +401,7 @@ def test_repo_input_handles_replaced_tracked_directories(assert [row["path"] for row in read_jsonl(output)] == expected[email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])-def test_repo_input_preserves_tracked_ignored_and_binary_files(- tmp_path: Path, command: str-) -> None:+def test_repo_input_preserves_tracked_ignored_and_binary_files(tmp_path: Path) -> None:repo = tmp_path / "repo"source = repo / "src"vendor = source / "vendor"@@ -486,16 +415,13 @@ def test_repo_input_preserves_tracked_ignored_and_binary_files(scopes.write_text(json.dumps(["src"]), encoding="utf-8")output = tmp_path / "scoped-source-input.jsonl"- run_repo_cli(command, repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))- expected = ["src/vendor/dependency.py"]- if command == "make-repo-scope-input":- expected.insert(0, "src/logo.png")+ expected = ["src/logo.png", "src/vendor/dependency.py"]assert [row["path"] for row in read_jsonl(output)] == expected[email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])-def test_repo_input_respects_ignored_directory_descendants(tmp_path: Path, command: str) -> None:+def test_repo_input_respects_ignored_directory_descendants(tmp_path: Path) -> None:repo = tmp_path / "repo"(repo / "src" / "node_modules").mkdir(parents=True)(repo / "src" / "nested").mkdir()@@ -510,7 +436,7 @@ def test_repo_input_respects_ignored_directory_descendants(tmp_path: Path, commascopes.write_text(json.dumps(["src"]), encoding="utf-8")output = tmp_path / "scoped-source-input.jsonl"- run_repo_cli(command, repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))assert {row["path"] for row in read_jsonl(output)} == {"src/handler.py",@@ -519,8 +445,7 @@ def test_repo_input_respects_ignored_directory_descendants(tmp_path: Path, comma}[email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])-def test_repo_input_keeps_explicitly_requested_ignored_file(tmp_path: Path, command: str) -> None:+def test_repo_input_keeps_explicitly_requested_ignored_file(tmp_path: Path) -> None:repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)(repo / ".gitignore").write_text(".env\n", encoding="utf-8")@@ -529,13 +454,12 @@ def test_repo_input_keeps_explicitly_requested_ignored_file(tmp_path: Path, commscopes.write_text(json.dumps(["src/.env"]), encoding="utf-8")output = tmp_path / "scoped-source-input.jsonl"- run_repo_cli(command, repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))assert [row["path"] for row in read_jsonl(output)] == ["src/.env"][email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])-def test_repo_input_uses_git_ignore_rules(tmp_path: Path, command: str) -> None:+def test_repo_input_uses_git_ignore_rules(tmp_path: Path) -> None:repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)initialize_repo(repo)@@ -546,14 +470,13 @@ def test_repo_input_uses_git_ignore_rules(tmp_path: Path, command: str) -> None:scopes.write_text(json.dumps(["src"]), encoding="utf-8")output = tmp_path / "scoped-source-input.jsonl"- run_repo_cli(command, repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))assert [row["path"] for row in read_jsonl(output)] == ["src/handler.py"][email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])def test_repo_input_falls_back_without_git_or_ripgrep(- tmp_path: Path, monkeypatch: pytest.MonkeyPatch, command: str+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)@@ -563,14 +486,13 @@ def test_repo_input_falls_back_without_git_or_ripgrep(output = tmp_path / "scoped-source-input.jsonl"monkeypatch.setenv("PATH", str(tmp_path / "missing-tools"))- run_repo_cli(command, repo, output, "--scopes-file", str(scopes))+ run_repo_cli("make-repo-scope-input", repo, output, "--scopes-file", str(scopes))assert [row["path"] for row in read_jsonl(output)] == ["src/handler.py"][email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])def test_repo_input_fails_closed_when_ignore_rules_cannot_be_applied(- tmp_path: Path, monkeypatch: pytest.MonkeyPatch, command: str+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)@@ -581,16 +503,17 @@ def test_repo_input_fails_closed_when_ignore_rules_cannot_be_applied(output = tmp_path / "scoped-source-input.jsonl"monkeypatch.setenv("PATH", str(tmp_path / "missing-tools"))- result = run_repo_cli(command, repo, output, "--scopes-file", str(scopes), check=False)+ result = run_repo_cli(+ "make-repo-scope-input", repo, output, "--scopes-file", str(scopes), check=False+ )assert result.returncode != 0assert "without Git or ripgrep" in result.stderrassert not output.exists()[email protected]("command", ["make-repo-scope-input", "make-repo-rank-input"])def test_repo_input_fails_closed_for_git_private_excludes_without_tools(- tmp_path: Path, monkeypatch: pytest.MonkeyPatch, command: str+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:repo = tmp_path / "repo"(repo / ".git" / "info").mkdir(parents=True)@@ -602,45 +525,15 @@ def test_repo_input_fails_closed_for_git_private_excludes_without_tools(output = tmp_path / "scoped-source-input.jsonl"monkeypatch.setenv("PATH", str(tmp_path / "missing-tools"))- result = run_repo_cli(command, repo, output, "--scopes-file", str(scopes), check=False)+ result = run_repo_cli(+ "make-repo-scope-input", repo, output, "--scopes-file", str(scopes), check=False+ )assert result.returncode != 0assert "without Git or ripgrep" in result.stderrassert not output.exists()-def test_make_repo_rank_input_keeps_explicit_binary_file_without_preview(tmp_path: Path) -> None:- repo = tmp_path / "repo"- repo.mkdir()- with (repo / "payload.bin").open("wb") as payload:- payload.write(b"header-without-a-nul" * 4096)- payload.truncate(256 * 1024 * 1024)- scopes = tmp_path / "target-paths.json"- scopes.write_text(json.dumps(["payload.bin"]), encoding="utf-8")- output = tmp_path / "rank_input.jsonl"-- run_repo_cli("make-repo-rank-input", repo, output, "--scopes-file", str(scopes))-- assert read_jsonl(output) == [{"path": "payload.bin", "area": "payload.bin", "preview": ""}]---def test_make_repo_rank_input_bounds_explicit_source_like_binary(tmp_path: Path) -> None:- repo = tmp_path / "repo"- source = repo / "src"- source.mkdir(parents=True)- with (source / "payload.py").open("wb") as payload:- payload.write(b"header-without-a-nul" * 4096)- payload.write(b"\0binary")- payload.truncate(256 * 1024 * 1024)- scopes = tmp_path / "target-paths.json"- scopes.write_text(json.dumps(["src", "src/payload.py"]), encoding="utf-8")- output = tmp_path / "rank_input.jsonl"-- run_repo_cli("make-repo-rank-input", repo, output, "--scopes-file", str(scopes))-- assert read_jsonl(output) == [{"path": "src/payload.py", "area": "src", "preview": ""}]--@pytest.mark.parametrize("mode", ["revisions", "staged", "unstaged"])def test_make_diff_rank_input_keeps_changed_and_deleted_text(tmp_path: Path, mode: str) -> None:repo = tmp_path / "repo"@@ -804,7 +697,7 @@ def test_make_diff_rank_input_combines_staged_and_unstaged_patch(tmp_path: Path)][email protected]("mode", ["repo", "explicit-file", "revisions", "local-patch"])[email protected]("mode", ["revisions", "local-patch"])def test_make_rank_input_decodes_bom_marked_utf16_source(tmp_path: Path, mode: str) -> None:repo = tmp_path / "repo"source_dir = repo / "src"@@ -825,24 +718,13 @@ def test_make_rank_input_decodes_bom_marked_utf16_source(tmp_path: Path, mode: s"src/utf8.ps1": source.strip(),}- if mode in {"repo", "explicit-file"}:- command = "make-repo-rank-input"- arguments = []- if mode == "explicit-file":- expected.update({"src/binary.ps1": "", "src/decoded-nul.ps1": ""})- scopes = tmp_path / "target-paths.json"- scopes.write_text(json.dumps(list(expected)), encoding="utf-8")- arguments.extend(["--scopes-file", str(scopes)])- else:- arguments.extend(["--scope", "src"])- else:- command = "make-diff-rank-input"- arguments = ["--base", base, "--mode", mode]- if mode == "revisions":- git(repo, "add", ".")- git(repo, "commit", "-qm", "add encoded source")- arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])- git(repo, "checkout", "-q", base)+ command = "make-diff-rank-input"+ arguments = ["--base", base, "--mode", mode]+ if mode == "revisions":+ git(repo, "add", ".")+ git(repo, "commit", "-qm", "add encoded source")+ arguments.extend(["--head", git(repo, "rev-parse", "HEAD")])+ git(repo, "checkout", "-q", base)run_repo_cli(command, repo, output, *arguments)diff --git a/sdk/typescript/tests-ts/deep-review-input.test.ts b/sdk/typescript/tests-ts/deep-review-input.test.tsindex 91946e237..8a71452a3 100644--- a/sdk/typescript/tests-ts/deep-review-input.test.ts+++ b/sdk/typescript/tests-ts/deep-review-input.test.ts@@ -14,7 +14,6 @@ import { tmpdir } from "node:os";import { join, relative } from "node:path";import { afterEach, describe, expect, spyOn, test } from "bun:test";import { PLUGIN_ROOT } from "./plugin-root.js";-import { runCommand } from "./support/shell.js";import { removeTemporaryDirectory } from "./support/temporary-directories.js";import { runTestInSubprocess } from "./support/test-subprocess.js";import { windowsHelperFixture } from "./windows-helper-command.js";@@ -33,13 +32,6 @@ const candidate = (path: string, area = "src") => ({area,preview: "source preview",});-const ranked = (path: string, score = 5, include = true, area = "src") => ({- path,- area,- score,- include,- reason: "runtime surface",-});function fixture() {const root = realpathSync(mkdtempSync(join(tmpdir(), "deep-review-input-")));roots.push(root);@@ -60,7 +52,6 @@ function read(path: string) {}function run(f: ReturnType<typeof fixture>,- selection: boolean,args: string[] = [],env = process.env,) {@@ -68,8 +59,8 @@ function run(node,[helper,- selection ? "select-deep-review-input" : "copy-deep-review-input",- selection ? "--rank-output" : "--rank-input",+ "copy-deep-review-input",+ "--rank-input",f.input,"--out",f.output,@@ -83,113 +74,28 @@ afterEach(async () => {});describe("deep-review worklists", () => {- test("selects rows with long internal whitespace without rescanning suffixes", async () => {- const f = fixture();- write(f.input, [- { ...ranked("a.py"), reason: "a" + " ".repeat(128000) + "b" },- ]);- const result = await runCommand(- node,- [- helper,- "select-deep-review-input",- "--rank-output",- f.input,- "--out",- f.output,- ],- { cwd: f.root, timeout: 5000 },- );- expect(result.status, result.stderr).toBe(0);- expect(read(f.output)).toEqual([{ path: "a.py", area: "src" }]);- });- test("copies all candidates in input order and selects included ranked rows", () => {+ test("copies every candidate in input order and preserves Unicode fields", () => {const f = fixture();- write(f.input, [candidate("a.py", "core"), candidate("b.py", "api")]);- expect(run(f, false).status).toBe(0);- expect(read(f.output)).toEqual([- { path: "a.py", area: "core" },- { path: "b.py", area: "api" },- ]);write(f.input, [- ranked("c.py", 8, true, "api"),- ranked("a.py", 10, true, "core"),- ranked("b.py", 8, true, "api"),- ranked("d.py", 2, false, "core"),+ candidate("𐀀.py", "é"),+ candidate("\ue000.py", "\udcff"),+ candidate("b.py", "api"),]);- const result = run(f, true, ["--top-percent", "67"]);+ const result = run(f);expect(result.status).toBe(0);- expect(result.stdout).toBe(- `Selected 2 of 3 rows into ${f.output}${newline}`,- );+ expect(result.stdout).toBe(`Copied 3 rows into ${f.output}${newline}`);expect(read(f.output)).toEqual([- { path: "a.py", area: "core" },- { path: "b.py", area: "api" },- ]);- });- test("honors explicit 20 percent and defaults to 100 percent", () => {- const f = fixture();- write(- f.input,- Array.from({ length: 5 }, (_, index) =>- ranked(`src/file_${index}.py`, 10 - index),- ),- );- expect(run(f, true, ["--top-percent", "20"]).status).toBe(0);- expect(read(f.output)).toEqual([{ path: "src/file_0.py", area: "src" }]);- expect(run(f, true).status).toBe(0);- expect(read(f.output)).toHaveLength(5);- });- test("falls back to all rows when workers exclude everything", () => {- const f = fixture();- write(f.input, [- ranked("b.py", 2, false, "api"),- ranked("a.py", 9, false, "core"),- ]);- expect(run(f, true).status).toBe(0);- expect(read(f.output)).toEqual([- { path: "a.py", area: "core" },+ { path: "𐀀.py", area: "é" },+ { path: "\ue000.py", area: "\udcff" },{ path: "b.py", area: "api" },]);});- test.each([false, true])(- "closes an empty worklist with an empty output (selection=%j)",- (selection) => {- const f = fixture();- write(f.input, []);- writeFileSync(f.output, "previous contents");- expect(run(f, selection).status).toBe(0);- expect(readFileSync(f.output)).toHaveLength(0);- },- );- test.each([- ["0", 1],- ["-10", 1],- ["1", 1],- ["200", 3],- ["+20", 1],- ] as const)(- "selects the requested integer percentage %s",- (percent, count) => {- const f = fixture();- write(f.input, [ranked("a.py"), ranked("b.py"), ranked("c.py")]);- expect(run(f, true, [`--top-percent=${percent}`]).status).toBe(0);- expect(read(f.output)).toHaveLength(count);- },- );- test("orders tied Unicode paths and preserves their contents", () => {+ test("closes an empty worklist with an empty output", () => {const f = fixture();- write(f.input, [- ranked("𐀀.py", 5, true, "é"),- ranked("\ue000.py", 5, true, "\udcff"),- ranked("\u007f.py", 5, true, "\t"),- ]);- expect(run(f, true).status).toBe(0);- expect(read(f.output)).toEqual([- { path: "\u007f.py", area: "\t" },- { path: "\ue000.py", area: "\udcff" },- { path: "𐀀.py", area: "é" },- ]);+ write(f.input, []);+ writeFileSync(f.output, "previous contents");+ expect(run(f).status).toBe(0);+ expect(readFileSync(f.output)).toHaveLength(0);});test.each([["\n", false],@@ -209,7 +115,7 @@ describe("deep-review worklists", () => {JSON.stringify(candidate("b.py")) +(terminated ? separator : ""),);- expect(run(f, false).status).toBe(0);+ expect(run(f).status).toBe(0);expect(read(f.output)).toEqual([{ path: "é.py", area: "雪" },{ path: "b.py", area: "src" },@@ -232,7 +138,7 @@ describe("deep-review worklists", () => {JSON.stringify(candidate("c.py")) +"\r",);- expect(run(f, false).status).toBe(0);+ expect(run(f).status).toBe(0);expect(read(f.output)).toEqual([{ path: "a.py", area: "src" },{ path: "b.py", area: "src" },@@ -290,62 +196,32 @@ describe("deep-review worklists", () => {}},);- const invalid: Array<[boolean, string | Buffer, string]> = [- [false, "\n", "blank JSONL rows are not allowed"],+ const invalid: Array<[string | Buffer, string]> = [+ ["\n", "blank JSONL rows are not allowed"],[- false,JSON.stringify(candidate("a.py")) + "\r\n\r",":2: blank JSONL rows are not allowed",],- [false, "\u001c\n", "invalid JSON"],- [false, "{}\n\n", "missing fields"],- [false, "{not json}\n", "invalid JSON"],- [false, "[]\n", "expected a JSON object"],+ ["\u001c\n", "invalid JSON"],+ ["{}\n\n", "missing fields"],+ ["{not json}\n", "invalid JSON"],+ ["[]\n", "expected a JSON object"],[- false,JSON.stringify({ ...candidate("a.py"), extra: true }),'unexpected fields ["extra"]',],- [false, JSON.stringify(candidate(" ")), "path must be a non-empty string"],+ [JSON.stringify(candidate(" ")), "path must be a non-empty string"],[- false,JSON.stringify({ ...candidate("a.py"), preview: 1 }),"preview must be a string",],[- false,JSON.stringify({ ...candidate("a.py"), area: null }),"area must be a string",],+ ["\ufeff" + JSON.stringify(candidate("a.py")), "invalid JSON"],+ [Buffer.from([0xff]), "encoded data"],[- true,- JSON.stringify({ ...ranked("a.py"), score: true }),- "score must be an integer",- ],- [- true,- '{"path":"a.py","area":"src","score":5.5,"include":true,"reason":"x"}',- "score must be an integer",- ],- [- true,- JSON.stringify(ranked("a.py", 11)),- "score must be from 1 through 10",- ],- [- true,- JSON.stringify({ ...ranked("a.py"), include: 1 }),- "include must be a boolean",- ],- [- true,- JSON.stringify({ ...ranked("a.py"), reason: "\t" }),- "reason must be a non-empty string",- ],- [false, "\ufeff" + JSON.stringify(candidate("a.py")), "invalid JSON"],- [false, Buffer.from([0xff]), "encoded data"],- [- false,Buffer.concat([Buffer.from(JSON.stringify(candidate("é.py")) + "\r\n"),Buffer.from([0xc3]),@@ -353,7 +229,6 @@ describe("deep-review worklists", () => {"encoded data",],[- false,Buffer.concat([Buffer.from('{"path":"a.py","area":"src","preview":"'),Buffer.from([0xed, 0xa0, 0x80]),@@ -362,50 +237,42 @@ describe("deep-review worklists", () => {"encoded data",],];- test.each(invalid)(- "rejects invalid worklist data (selection=%j, data=%j)",- (selection, text, message) => {- const f = fixture();- writeFileSync(f.input, text);- writeFileSync(f.output, "preserve existing output");- const result = run(f, selection);- expect(result.status).toBe(1);- expect(result.stderr).toContain(message);- expect(readFileSync(f.output, "utf8")).toBe("preserve existing output");- },- );- test.each([false, true])(- "rejects repeated paths before output (selection=%j)",- (selection) => {- const f = fixture();- write(- f.input,- selection- ? [ranked("a-\u007f\u009b.py"), ranked("a-\u007f\u009b.py")]- : [candidate("a-\u007f\u009b.py"), candidate("a-\u007f\u009b.py")],- );- const result = run(f, selection);- expect(result.status).toBe(1);- expect(result.stderr).toContain(- 'contains duplicate paths: ["a-\\u007f\\u009b.py"]',- );- expect(result.stderr).not.toMatch(/[\u007f-\u009f]/u);- expect(existsSync(f.output)).toBe(false);- },- );+ test.each(invalid)("rejects invalid worklist data %j", (text, message) => {+ const f = fixture();+ writeFileSync(f.input, text);+ writeFileSync(f.output, "preserve existing output");+ const result = run(f);+ expect(result.status).toBe(1);+ expect(result.stderr).toContain(message);+ expect(readFileSync(f.output, "utf8")).toBe("preserve existing output");+ });+ test("rejects repeated paths before output", () => {+ const f = fixture();+ write(f.input, [+ candidate("a-\u007f\u009b.py"),+ candidate("a-\u007f\u009b.py"),+ ]);+ const result = run(f);+ expect(result.status).toBe(1);+ expect(result.stderr).toContain(+ 'contains duplicate paths: ["a-\\u007f\\u009b.py"]',+ );+ expect(result.stderr).not.toMatch(/[\u007f-\u009f]/u);+ expect(existsSync(f.output)).toBe(false);+ });test("accepts duplicate JSON object keys and empty area/preview as before", () => {const f = fixture();writeFileSync(f.input,'{"path":"unused.py","path":"a.py","area":"","preview":""}',);- expect(run(f, false).status).toBe(0);+ expect(run(f).status).toBe(0);expect(read(f.output)).toEqual([{ path: "a.py", area: "" }]);});test("allows replacing the input after all rows have been read", () => {const f = fixture();write(f.input, [candidate("b.py"), candidate("a.py")]);- expect(run({ ...f, output: f.input }, false).status).toBe(0);+ expect(run({ ...f, output: f.input }).status).toBe(0);expect(read(f.input)).toEqual([{ path: "b.py", area: "src" },{ path: "a.py", area: "src" },@@ -417,7 +284,6 @@ describe("deep-review worklists", () => {const env = { ...process.env, HOME: f.root, USERPROFILE: f.root };const result = run({ ...f, input: "~/input.jsonl", output: "./nested/./output.jsonl" },- false,[],env,);@@ -427,24 +293,20 @@ describe("deep-review worklists", () => {);expect(read(join(f.root, "nested", "output.jsonl"))).toHaveLength(1);});- test.each([false, true])(- "removes pathlib dot and empty components from input and output (selection=%j)",- (selection) => {- const f = fixture();- write(f.input, [selection ? ranked("a.py") : candidate("a.py")]);- for (const suffix of ["/.", "///", "/./."]) {- const result = run(- { ...f, input: f.input + suffix, output: f.output + suffix },- selection,- );- expect(result.status, result.stderr).toBe(0);- expect(result.stdout).toBe(- `${selection ? "Selected 1 of 1" : "Copied 1"} rows into ${f.output}${newline}`,- );- expect(read(f.output)).toEqual([{ path: "a.py", area: "src" }]);- }- },- );+ test("removes pathlib dot and empty components from input and output", () => {+ const f = fixture();+ write(f.input, [candidate("a.py")]);+ for (const suffix of ["/.", "///", "/./."]) {+ const result = run({+ ...f,+ input: f.input + suffix,+ output: f.output + suffix,+ });+ expect(result.status, result.stderr).toBe(0);+ expect(result.stdout).toBe(`Copied 1 rows into ${f.output}${newline}`);+ expect(read(f.output)).toEqual([{ path: "a.py", area: "src" }]);+ }+ });test("reports missing input and preserves argument parsing status", () => {const f = fixture();writeFileSync(f.output, "previous output\n");@@ -454,29 +316,15 @@ describe("deep-review worklists", () => {symlinkSync(loop, loop);inputs.push(loop);}- for (const selection of [false, true]) {- for (const input of inputs) {- const result = run({ ...f, input }, selection);- expect(result.status).toBe(1);- expect(result.stderr).toBe(- `Rank ${selection ? "output" : "input"} missing: ${input}${newline}`,- );- }- expect(readFileSync(f.output, "utf8")).toBe("previous output\n");+ for (const input of inputs) {+ const result = run({ ...f, input });+ expect(result.status).toBe(1);+ expect(result.stderr).toBe(`Rank input missing: ${input}${newline}`);}- write(f.input, [ranked("a.py")]);- expect(run(f, true, ["--top-percent=20"]).status).toBe(0);- expect(run(f, true, ["--top-percent", "20"]).status).toBe(0);- for (const args of [- ["--top-percent", "1.0"],- ["--top-percent"],- ["--unknown"],- ["extra"],- ["--top-percent", "1__0"],- ["--top-percent", "\u001c20"],- ])- expect(run(f, true, args).status).toBe(2);- expect(run(f, true, ["--help"]).stdout).toContain("Defaults to 100");+ expect(readFileSync(f.output, "utf8")).toBe("previous output\n");+ for (const args of [["--unknown"], ["extra"], ["--rank-input"]])+ expect(run(f, args).status).toBe(2);+ expect(run(f, ["--help"]).stdout).toContain("copy-deep-review-input");});test.skipIf(process.platform === "win32")("escapes controls in filesystem errors but preserves other command text",@@ -484,31 +332,25 @@ describe("deep-review worklists", () => {const f = fixture();f.input = join(f.root, "input-\x1b[2J.jsonl");f.output = join(f.root, "output-\x1b[2J.jsonl");- for (const selection of [false, true]) {- const missing = run(f, selection);- expect(missing.status).toBe(1);- expect(missing.stderr).toBe(- `Rank ${selection ? "output" : "input"} missing: ${f.input}\n`,- );- writeFileSync(f.input, "{}\n");- const invalid = run(f, selection);- expect(invalid.status).toBe(1);- expect(invalid.stderr).toStartWith(`${f.input}:1: missing fields`);- write(f.input, [selection ? ranked("a.py") : candidate("a.py")]);- mkdirSync(f.output);- const collision = run(f, selection);- expect(collision.status).toBe(1);- expect(collision.stderr).toMatch(/\\(?:x1b|u001b)\[2J/);- expect(collision.stderr).not.toContain("\x1b");- rmSync(f.output, { recursive: true });- const success = run(f, selection);- expect(success.status).toBe(0);- expect(success.stdout).toBe(- `${selection ? "Selected 1 of 1" : "Copied 1"} rows into ${f.output}\n`,- );- rmSync(f.input);- rmSync(f.output);- }+ const missing = run(f);+ expect(missing.status).toBe(1);+ expect(missing.stderr).toBe(`Rank input missing: ${f.input}\n`);+ writeFileSync(f.input, "{}\n");+ const invalid = run(f);+ expect(invalid.status).toBe(1);+ expect(invalid.stderr).toStartWith(`${f.input}:1: missing fields`);+ write(f.input, [candidate("a.py")]);+ mkdirSync(f.output);+ const collision = run(f);+ expect(collision.status).toBe(1);+ expect(collision.stderr).toMatch(/\\(?:x1b|u001b)\[2J/);+ expect(collision.stderr).not.toContain("\x1b");+ rmSync(f.output, { recursive: true });+ const success = run(f);+ expect(success.status).toBe(0);+ expect(success.stdout).toBe(`Copied 1 rows into ${f.output}\n`);+ rmSync(f.input);+ rmSync(f.output);},);test.skipIf(process.platform === "win32")(@@ -521,7 +363,7 @@ describe("deep-review worklists", () => {write(f.input, [candidate("a.py")]);const inputLink = join(f.root, "input-link");symlinkSync(f.input, inputLink);- expect(run({ ...f, input: inputLink }, false).status).toBe(0);+ expect(run({ ...f, input: inputLink }).status).toBe(0);expect(read(target)).toHaveLength(1);expect(statSync(target).mode & 0o777).toBe(0o640);},@@ -536,22 +378,17 @@ describe("deep-review worklists", () => {write(f.input, [candidate("wrong.py")]);writeFileSync(f.output, "output sentinel");const input = join(child, "input.jsonl");- for (const selection of [false, true]) {- write(input, [selection ? ranked("right.py") : candidate("right.py")]);- const result = run(- {- ...f,- input: "link/../input.jsonl/.",- output: "link/../output.jsonl/.",- },- selection,- );- expect(result.status, result.stderr).toBe(0);- expect(read(join(child, "output.jsonl"))).toEqual([- { path: "right.py", area: "src" },- ]);- expect(readFileSync(f.output, "utf8")).toBe("output sentinel");- }+ write(input, [candidate("right.py")]);+ const result = run({+ ...f,+ input: "link/../input.jsonl/.",+ output: "link/../output.jsonl/.",+ });+ expect(result.status, result.stderr).toBe(0);+ expect(read(join(child, "output.jsonl"))).toEqual([+ { path: "right.py", area: "src" },+ ]);+ expect(readFileSync(f.output, "utf8")).toBe("output sentinel");},);test.skipIf(process.platform === "win32")(diff --git a/sdk/typescript/tests-ts/rank-pool.test.ts b/sdk/typescript/tests-ts/rank-pool.test.tsdeleted file mode 100644index 32fe5ad5d..000000000--- a/sdk/typescript/tests-ts/rank-pool.test.ts+++ /dev/null@@ -1,689 +0,0 @@-import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js";-import { spawnSync } from "node:child_process";-import { createHash } from "node:crypto";-import {- existsSync,- mkdirSync,- readFileSync,- readdirSync,- rmSync,- statSync,- symlinkSync,- writeFileSync,-} from "node:fs";-import { join, sep } from "node:path";-import { afterEach, describe, expect, test } from "bun:test";-import { PLUGIN_ROOT } from "./plugin-root.js";--const node = Bun.which("node")!;-const helper = join(PLUGIN_ROOT, "mcp", "helpers.mjs");-const newline = process.platform === "win32" ? "\r\n" : "\n";-const roots = createTemporaryDirectoriesSync(true);--const shardName = (index: number, output = false) =>- `rank-shard-${String(index).padStart(4, "0")}.${output ? "output" : "input"}.jsonl`;-const row = (index: number) => ({- path: `src/file_${index}.py`,- area: "src",- preview: `value = ${index}`,-});-const ranked = (index: number) => ({- path: row(index).path,- area: "src",- score: 5,- include: true,- reason: "runtime surface",-});-const expectedPlan = {- ranking_worker_count: 2,- schema_version: 1,- shard_count: 5,- strategy: "round_robin",- workers: [- {- input_shards: [1, 3, 5].map((index) => shardName(index)),- output_shards: [1, 3, 5].map((index) => shardName(index, true)),- slot: 1,- },- {- input_shards: [2, 4].map((index) => shardName(index)),- output_shards: [2, 4].map((index) => shardName(index, true)),- slot: 2,- },- ],-};-type Plan = typeof expectedPlan;-type Worker = Plan["workers"][number];--function fixture(count = 5) {- const root = roots.create("rank-pool-");- const directory = join(root, "rank_shards");- mkdirSync(directory);- for (let index = 1; index <= count; index++)- writeFileSync(- join(directory, shardName(index)),- JSON.stringify(row(index)) + "\n",- );- return { root, directory, plan: join(root, "assignments.json") };-}-type Fixture = ReturnType<typeof fixture>;-function run(f: Fixture, command: string, args: string[], env = process.env) {- return spawnSync(node, [helper, command, ...args], {- cwd: f.root,- env,- encoding: "utf8",- input: "stdin is not a plan",- });-}-function make(f: Fixture, slots = "2") {- return run(f, "make-rank-pool-plan", [- "--shard-dir",- f.directory,- "--usable-worker-slots",- slots,- "--out",- f.plan,- ]);-}-function validate(f: Fixture, slot?: string, extra: string[] = []) {- return run(- f,- slot === undefined ? "validate-rank-pool" : "validate-rank-worker",- [- "--plan",- f.plan,- "--shard-dir",- f.directory,- ...(slot === undefined ? [] : ["--slot", slot]),- ...extra,- ],- );-}-function plan(f: Fixture): Plan {- return JSON.parse(readFileSync(f.plan, "utf8")) as Plan;-}-function change(f: Fixture, edit: (value: Plan) => void) {- const value = plan(f);- edit(value);- writeFileSync(f.plan, JSON.stringify(value));-}-function complete(f: Fixture, slots?: number[]) {- for (const worker of plan(f).workers) {- if (slots && !slots.includes(worker.slot)) continue;- for (const name of worker.output_shards) {- const index = Number(name.match(/([0-9]+)\.output/u)![1]);- writeFileSync(- join(f.directory, name),- JSON.stringify(ranked(index)) + "\n",- );- }- }-}-afterEach(roots.cleanup);--describe("rank pool helpers", () => {- test("writes stable sorted JSON and assigns every shard round-robin", () => {- const f = fixture();- expect(make(f).stdout).toBe(- `Assigned 5 rank shards to 2 ranking workers in ${f.plan}${newline}`,- );- expect(readFileSync(f.plan, "utf8")).toBe(- (JSON.stringify(expectedPlan, null, 2) + "\n").replaceAll("\n", newline),- );- const first = readFileSync(f.plan);- expect(make(f).status).toBe(0);- expect(readFileSync(f.plan)).toEqual(first);- complete(f);- expect(validate(f).stdout).toBe(- `Validated 2 ranking workers, 5 shards, and 5 ranking rows${newline}`,- );- });-- test.each([- [0, "6", 0],- [2, "8", 2],- [8, "12", 6],- ] as const)(- "caps %i shards with %s usable slots at %i workers",- (count, slots, workers) => {- const f = fixture(count);- expect(make(f, slots).status).toBe(0);- expect(plan(f).ranking_worker_count).toBe(workers);- expect(plan(f).workers).toHaveLength(workers);- complete(f);- expect(validate(f).status).toBe(0);- if (count === 0)- expect(validate(f, "1").stderr).toBe(- `--slot must be at most 0${newline}`,- );- },- );-- test("validates one worker independently and binds its receipt to raw plan and output bytes", () => {- const f = fixture();- make(f);- complete(f, [1]);- const names = [1, 3, 5].map((index) => shardName(index, true));- const output = join(f.directory, names[1]!);- writeFileSync(output, JSON.stringify(ranked(3), null, 0) + "\r\n");- const bytes = Buffer.concat([- Buffer.from([0xef, 0xbb, 0xbf]),- readFileSync(f.plan),- ]);- writeFileSync(f.plan, bytes);- const digest = createHash("sha256");- for (const name of names)- digest- .update(name)- .update("\0")- .update(readFileSync(join(f.directory, name)))- .update("\0");- const result = validate(f, "1");- expect(result.status).toBe(0);- expect(result.stderr).toBe("");- expect(result.stdout).toBe(- "RANK_WORKER_RECEIPT " +- JSON.stringify({- output_shards: 3,- outputs_sha256: digest.digest("hex"),- plan_sha256: createHash("sha256").update(bytes).digest("hex"),- ranking_worker_count: 2,- rows: 3,- schema_version: 1,- slot: 1,- status: "complete",- }) +- newline,- );- expect(existsSync(join(f.directory, shardName(2, true)))).toBe(false);- expect(validate(f).stderr).toContain("missing output shards");- const second = run(f, "validate-rank-worker", [- "--plan",- f.plan,- "--shard-dir",- f.directory,- "--slot",- "2",- ]);- expect(second.status).toBe(1);- expect(second.stdout).toBe("");- });-- test("accepts completed slots without requiring worker receipts or unrelated outputs", () => {- const f = fixture();- make(f);- complete(f);- writeFileSync(join(f.directory, "unrelated.txt"), "untouched");- expect(validate(f, "1").status).toBe(0);- expect(validate(f, "2").status).toBe(0);- expect(validate(f).status).toBe(0);- writeFileSync(- join(f.directory, shardName(2, true)),- "invalid unassigned output",- );- expect(validate(f, "1").status).toBe(0);- expect(validate(f).status).toBe(1);- });-- test.each(["missing", "invalid JSON", "duplicate", "area", "paths"])(- "rejects %s assigned output",- (kind) => {- const f = fixture();- make(f);- complete(f);- const output = join(f.directory, shardName(3, true));- if (kind === "missing") rmSync(output);- else if (kind === "invalid JSON") writeFileSync(output, "{bad json}\n");- else if (kind === "duplicate")- writeFileSync(output, (JSON.stringify(ranked(3)) + "\n").repeat(2));- else- writeFileSync(- output,- JSON.stringify({- ...ranked(3),- [kind === "area" ? "area" : "path"]: "different",- }) + "\n",- );- expect(validate(f, "1").status).toBe(1);- expect(validate(f, "1").stdout).toBe("");- expect(validate(f).status).toBe(1);- },- );-- test("reports both missing and unexpected output shards", () => {- const f = fixture();- make(f);- complete(f);- rmSync(join(f.directory, shardName(5, true)));- writeFileSync(join(f.directory, shardName(9999, true)), "");- expect(validate(f).stderr).toBe(- `Rank pool outputs are incomplete; missing output shards=["${shardName(5, true)}"]; unexpected output shards=["${shardName(9999, true)}"]${newline}`,- );- });-- test.each(["input", "output"])(- "rejects a misplaced %s shard before loading the plan",- (kind) => {- const f = fixture();- const name = shardName(1, kind === "output");- writeFileSync(join(f.root, name), "");- expect(validate(f).stderr).toContain(`misplaced=["${name}"]`);- expect(validate(f, "1").status).toBe(1);- // Plan creation has no misplaced-artifact check in the original helper.- expect(make(f).status).toBe(0);- },- );-- test("requires the sibling shard directory and resolves aliases with parent traversal", () => {- const f = fixture();- mkdirSync(join(f.root, "other"));- const bad = { ...f, directory: join(f.root, "other") };- expect(make(bad).stderr).toContain(- "must be the assignment plan's sibling rank_shards directory",- );- expect(validate(bad, "1").stderr).toContain(- "must be the assignment plan's sibling rank_shards directory",- );- symlinkSync(- f.directory,- join(f.root, "alias"),- process.platform === "win32" ? "junction" : "dir",- );- expect(make({ ...f, directory: join(f.root, "alias") }).status).toBe(0);- complete(f);- expect(validate({ ...f, plan: f.plan + sep + "." }).status).toBe(0);- expect(- validate({- ...f,- plan: f.directory + sep + ".." + sep + "assignments.json",- }).status,- ).toBe(0);- });-- test("rejects looping shard aliases without replacing the plan", () => {- const f = fixture();- const loop = join(- f.root,- process.platform === "win32" ? "loop" : "loop-\u001b[2J",- );- symlinkSync(loop, loop, process.platform === "win32" ? "junction" : "dir");- writeFileSync(f.plan, "preserve plan");- const alias = { ...f, directory: loop };- for (const result of [make(alias), validate(alias), validate(alias, "1")]) {- expect(result.status).toBe(1);- if (process.platform !== "win32") {- expect(result.stderr).toContain("ELOOP");- expect(result.stderr).toMatch(/loop-\\(?:x1b|u001b)\[2J/);- expect(result.stderr).not.toContain("\u001b");- }- }- expect(readFileSync(f.plan, "utf8")).toBe("preserve plan");- });-- test("checks shard existence and names before loading a missing or malformed plan", () => {- const f = fixture(0);- expect(validate(f).stderr).toBe(- `Rank pool plan missing: ${f.plan}${newline}`,- );- writeFileSync(f.plan, "bad");- writeFileSync(join(f.directory, "rank-shard-001.input.jsonl"), "");- expect(validate(f).stderr).toContain("contiguous canonical names");- expect(make(f).status).toBe(1);- expect(readFileSync(f.plan, "utf8")).toBe("bad");- rmSync(f.directory, { recursive: true });- expect(make(f).stderr).toContain("Rank shard directory missing");- });-- test.skipIf(process.platform === "win32")(- "escapes filesystem errors without changing custom plan diagnostics or output",- () => {- const f = fixture(0);- f.plan = join(f.root, "plan-\u001b[2J.json");- symlinkSync(f.plan, f.plan);- for (const slot of [undefined, "1"]) {- const missing = validate(f, slot);- expect(missing.status).toBe(1);- expect(missing.stderr).toBe(`Rank pool plan missing: ${f.plan}\n`);- }- rmSync(f.plan);- mkdirSync(f.plan);- const collision = make(f);- expect(collision.status).toBe(1);- expect(collision.stderr).toMatch(/\\(?:x1b|u001b)\[2J/);- expect(collision.stderr).not.toContain("\u001b");- rmSync(f.plan, { recursive: true });- writeFileSync(f.plan, "bad");- expect(validate(f).stderr).toStartWith(`${f.plan}: invalid JSON:`);- const success = make(f);- expect(success.status).toBe(0);- expect(success.stdout).toBe(- `Assigned 0 rank shards to 0 ranking workers in ${f.plan}\n`,- );- },- );-- test.each([- [- "root fields",- (p: Plan) => {- Object.assign(p, { extra: true });- },- 'unexpected=["extra"]',- ],- [- "version type",- (p: Plan) => {- Object.assign(p, { schema_version: true });- },- "schema_version must be an integer",- ],- [- "version",- (p: Plan) => {- p.schema_version = 2;- },- "schema_version must be 1",- ],- [- "strategy",- (p: Plan) => {- p.strategy = "other";- },- "strategy must be round_robin",- ],- [- "shard count",- (p: Plan) => {- p.shard_count = 4;- },- "shard_count does not match",- ],- [- "worker count",- (p: Plan) => {- p.ranking_worker_count = 0;- },- "must be at least 1",- ],- [- "workers type",- (p: Plan) => {- Object.assign(p, { workers: {} });- },- "exactly 2 worker assignments",- ],- [- "worker object",- (p: Plan) => {- p.workers[0] = null as unknown as Worker;- },- "must be a JSON object",- ],- [- "worker fields",- (p: Plan) => {- Object.assign(p.workers[0]!, { extra: true });- },- 'unexpected=["extra"]',- ],- [- "slot",- (p: Plan) => {- p.workers[0]!.slot = 2;- },- ".slot must be 1",- ],- [- "empty assignments",- (p: Plan) => {- p.workers[0]!.input_shards = [];- },- "must be a non-empty list",- ],- [- "empty name",- (p: Plan) => {- p.workers[0]!.input_shards[0] = "";- },- "entries must be non-empty strings",- ],- [- "unequal assignments",- (p: Plan) => {- p.workers[0]!.input_shards.pop();- },- "lengths must match",- ],- [- "output name",- (p: Plan) => {- p.workers[0]!.output_shards[0] = "wrong";- },- "do not match its input_shards",- ],- ] as const)("rejects tampered %s", (_name, edit, message) => {- const f = fixture();- make(f);- change(f, edit);- const result = validate(f);- expect(result.status).toBe(1);- expect(result.stderr).toContain(message);- expect(validate(f, "1").stdout).toBe("");- });-- test.each([- [- "missing",- [1, 5],- 'pool plan must assign each input shard exactly once; missing=["rank-shard-0003.input.jsonl"]; duplicates=[]; unexpected=[]',- ],- [- "duplicate",- [1, 1, 5],- 'pool plan must assign each input shard exactly once; missing=["rank-shard-0003.input.jsonl"]; duplicates=["rank-shard-0001.input.jsonl"]; unexpected=[]',- ],- [- "unexpected",- [1, 6, 5],- 'pool plan must assign each input shard exactly once; missing=["rank-shard-0003.input.jsonl"]; duplicates=[]; unexpected=["rank-shard-0006.input.jsonl"]',- ],- [- "reordered",- [3, 1, 5],- "worker slot 1 does not match the deterministic round_robin assignment",- ],- ] as const)(- "preserves diagnostics for %s assignments",- (_name, indexes, message) => {- const f = fixture();- make(f);- change(f, (p) => {- p.workers[0]!.input_shards = indexes.map((index) => shardName(index));- p.workers[0]!.output_shards = indexes.map((index) =>- shardName(index, true),- );- });- for (const slot of [undefined, "1"]) {- const result = validate(f, slot);- expect(result.status).toBe(1);- expect(result.stdout).toBe("");- expect(result.stderr).toBe(`${f.plan}: ${message}${newline}`);- }- },- );-- test.each(["schema", "outputs"])(- "checks every worker's %s before aggregate assignments",- (invalid) => {- const f = fixture();- make(f);- change(f, (p) => {- p.workers[0]!.input_shards[1] = shardName(1);- p.workers[0]!.output_shards[1] = shardName(1, true);- if (invalid === "schema") p.workers[1]!.slot = 1;- else p.workers[1]!.output_shards[0] = shardName(1, true);- });- const message =- invalid === "schema"- ? "slot must be 2"- : "output_shards do not match its input_shards";- for (const slot of [undefined, "1"]) {- const result = validate(f, slot);- expect(result.status).toBe(1);- expect(result.stdout).toBe("");- expect(result.stderr).toBe(- `${f.plan}: workers[1].${message}${newline}`,- );- }- },- );-- test.each(["{bad}", "[]", "null", "1", "true", "", "{} trailing"])(- "rejects malformed or nonobject plan %s",- (text) => {- const f = fixture();- writeFileSync(f.plan, text);- expect(validate(f).status).toBe(1);- expect(validate(f).stdout).toBe("");- },- );-- test("rejects malformed UTF-8 even in an overwritten plan value", () => {- const f = fixture(0);- make(f);- const rest = readFileSync(f.plan, "utf8").slice(1);- for (const bytes of [[0xff], [0xed, 0xa0], [0xed, 0xa0, 0x41]]) {- writeFileSync(- f.plan,- Buffer.concat([- Buffer.from('{"strategy":"'),- Buffer.from(bytes),- Buffer.from('",' + rest),- ]),- );- const result = validate(f);- expect(result.status).toBe(1);- expect(result.stdout).toBe("");- }- });-- test("preserves duplicate-key last value and validates integer field values", () => {- const f = fixture(0);- make(f);- const text = readFileSync(f.plan, "utf8");- writeFileSync(- f.plan,- text.replace(- '"schema_version": 1',- '"schema_version": 2, "schema_version": 1',- ),- );- expect(validate(f).status).toBe(0);- for (const value of ["1.0", "1e0"]) {- writeFileSync(- f.plan,- text.replace('"schema_version": 1', `"schema_version": ${value}`),- );- expect(validate(f).status).toBe(0);- }- for (const value of ["true", "1.5", "-1"]) {- writeFileSync(- f.plan,- text.replace('"schema_version": 1', `"schema_version": ${value}`),- );- expect(validate(f).status).toBe(1);- expect(validate(f).stderr).toContain(- "schema_version must be an integer of at least 1",- );- }- });-- test.each(["+20", "20"])(- "accepts decimal worker count %s with full option names",- (value) => {- const f = fixture(1);- expect(make(f, value).status).toBe(0);- complete(f);- expect(validate(f, "1", ["--slot=+1"]).status).toBe(0);- },- );-- test("preserves argument failure statuses and validation order", () => {- const f = fixture();- for (const value of ["0"])- expect(make({ ...f, directory: "missing" }, value).stderr).toBe(- `--usable-worker-slots must be at least 1${newline}`,- );- for (const value of ["1.0", "1__0", "\u001c20"])- expect(make(f, value).status).toBe(2);- for (const command of [- "make-rank-pool-plan",- "validate-rank-worker",- "validate-rank-pool",- ]) {- expect(run(f, command, []).status).toBe(2);- expect(run(f, command, ["--help"]).status).toBe(0);- }- expect(validate(f, "0").stderr).toContain(f.plan);- make(f);- expect(validate(f, "0").stderr).toBe(- `--slot must be an integer of at least 1${newline}`,- );- expect(validate(f, "3").stderr).toBe(`--slot must be at most 2${newline}`);- expect(validate(f, "1", ["--s", "1"]).status).toBe(2);- });-- test("preserves literal dash paths, output aliases, and existing file permissions", () => {- const f = fixture(0);- const dash = { ...f, directory: "rank_shards", plan: "-" };- writeFileSync(join(f.root, "-"), "previous", { mode: 0o640 });- expect(make(dash).status).toBe(0);- expect(validate(dash).status).toBe(0);- if (process.platform !== "win32")- expect(statSync(join(f.root, "-")).mode & 0o777).toBe(0o640);- symlinkSync(- join(f.root, "-"),- f.plan,- process.platform === "win32" ? "file" : undefined,- );- expect(make(f).status).toBe(0);- expect(readFileSync(f.plan)).toEqual(readFileSync(join(f.root, "-")));- expect(readdirSync(f.directory)).toHaveLength(0);- });-- test.skipIf(process.platform === "win32")(- "uses raw POSIX bytes for plan and shard paths",- () => {- const f = fixture(0);- const pathBytes =- process.platform === "darwin" ? Buffer.from("é") : Buffer.from([255]);- const suffix = Array.from(- pathBytes,- (byte) => `\\${byte.toString(8).padStart(3, "0")}`,- ).join("");- const raw = Buffer.concat([Buffer.from(f.root + "/"), pathBytes]);- mkdirSync(raw);- mkdirSync(Buffer.concat([raw, Buffer.from("/rank_shards")]));- const launcher = join(- PLUGIN_ROOT,- "scripts",- "launch_codex_security_mcp",- );- const result = spawnSync(- "bash",- [- "-c",- `r="$1"/$(printf '${suffix}'); "$2" --helper make-rank-pool-plan --shard-dir "$r/rank_shards" --usable-worker-slots 1 --out "$r/plan.json"`,- "rank-pool",- f.root,- launcher,- ],- { env: { ...process.env, CODEX_MCP_NODE_PATH: node } },- );- expect(result.status).toBe(0);- const output = Buffer.concat([raw, Buffer.from("/plan.json")]);- expect(JSON.parse(readFileSync(output, "utf8")).shard_count).toBe(0);- expect(result.stdout.includes(output)).toBe(true);- },- );-});diff --git a/sdk/typescript/tests-ts/rank-shards.test.ts b/sdk/typescript/tests-ts/rank-shards.test.tsdeleted file mode 100644index 086760723..000000000--- a/sdk/typescript/tests-ts/rank-shards.test.ts+++ /dev/null@@ -1,633 +0,0 @@-import { createTemporaryDirectoriesSync } from "./support/temporary-directories.js";-import { spawnSync } from "node:child_process";-import {- chmodSync,- existsSync,- mkdirSync,- readFileSync,- readdirSync,- renameSync,- rmSync,- statSync,- symlinkSync,- writeFileSync,-} from "node:fs";-import { dirname, join, sep } from "node:path";-import { afterEach, describe, expect, test } from "bun:test";-import { PLUGIN_ROOT } from "./plugin-root.js";--const node = Bun.which("node")!;-const helper = join(PLUGIN_ROOT, "mcp", "helpers.mjs");-const roots = createTemporaryDirectoriesSync(true);-const newline = process.platform === "win32" ? "\r\n" : "\n";-type Row = Record<string, unknown>;-const candidate = (path: string, area = "src") => ({- path,- area,- preview: "source preview",-});-const ranked = (row: Row) => ({- path: row["path"],- area: row["area"],- score: 5,- include: true,- reason: "runtime surface",-});-function fixture() {- const root = roots.create("rank-shards-");- return {- root,- input: join(root, "input.jsonl"),- directory: join(root, "shards"),- output: join(root, "output.jsonl"),- };-}-type Fixture = ReturnType<typeof fixture>;-function write(path: string, rows: Row[]) {- mkdirSync(dirname(path), { recursive: true });- writeFileSync(path, rows.map((row) => JSON.stringify(row) + "\n").join(""));-}-function read(path: string): Row[] {- const text = readFileSync(path, "utf8").trim();- return text === ""- ? []- : text.split(/\r?\n/u).map((line) => JSON.parse(line) as Row);-}-function run(f: Fixture, command: string, args: string[], env = process.env) {- return spawnSync(node, [helper, command, ...args], {- cwd: f.root,- encoding: "utf8",- env,- input: "stdin is not a worklist",- });-}-function make(f: Fixture, args: string[] = []) {- return run(f, "make-rank-shards", [- "--rank-input",- f.input,- "--out-dir",- f.directory,- ...args,- ]);-}-function merge(f: Fixture, args: string[] = []) {- return run(f, "merge-rank-outputs", [- "--rank-input",- f.input,- "--shard-dir",- f.directory,- "--out",- f.output,- ...args,- ]);-}-function shard(f: Fixture, index: number, output = false) {- return join(- f.directory,- `rank-shard-${String(index).padStart(4, "0")}.${output ? "output" : "input"}.jsonl`,- );-}-function validate(f: Fixture, index = 1) {- return run(f, "validate-rank-shard", [- "--input",- shard(f, index),- "--output",- shard(f, index, true),- ]);-}-function complete(f: Fixture) {- for (const name of readdirSync(f.directory).filter((name) =>- name.endsWith(".input.jsonl"),- ))- write(- f.directory + sep + name.replace(".input.", ".output."),- read(f.directory + sep + name)- .reverse()- .map(ranked),- );-}-afterEach(roots.cleanup);--describe("rank shard helpers", () => {- test.skipIf(process.platform !== "win32")(- "joins shard names beneath Windows roots without changing their path kind",- async () => {- const { childPath } = (await import(- new URL(- "../../../plugins/codex-security/mcp-app/src/helpers/rank-shards.ts",- import.meta.url,- ).href- )) as { childPath: (directory: string, name: string) => string };- const name = "rank-shard-0001.input.jsonl";- for (const [directory, prefix] of [- ["/", "\\"],- ["\\", "\\"],- ["C:", "C:"],- ["C:\\", "C:\\"],- ["\\\\server\\share", "\\\\server\\share\\"],- ["\\\\?\\C:\\", "\\\\?\\C:\\"],- ["\\\\?\\UNC\\server\\share\\", "\\\\?\\UNC\\server\\share\\"],- ]) {- expect(childPath(directory!, name)).toBe(prefix! + name);- }- },- );-- test("partitions deterministically at the default 150 rows and refuses existing shards", () => {- const f = fixture();- const rows = Array.from({ length: 312 }, (_, index) =>- candidate(`src/file_${index}.py`),- );- write(f.input, rows);- expect(make(f).stdout).toBe(- `Wrote 3 rank shards to ${f.directory}${newline}`,- );- expect(readdirSync(f.directory).sort()).toEqual(- [1, 2, 3].map((index) => `rank-shard-000${index}.input.jsonl`),- );- expect([1, 2, 3].map((index) => read(shard(f, index)).length)).toEqual([- 150, 150, 12,- ]);- expect([1, 2, 3].flatMap((index) => read(shard(f, index)))).toEqual(rows);- const original = readFileSync(shard(f, 1));- const second = make(f);- expect(second.status).toBe(1);- expect(second.stderr).toContain("already contains shard files");- expect(readFileSync(shard(f, 1))).toEqual(original);- });-- test.skipIf(process.platform === "win32")(- "preserves existing shards alongside Unicode and raw-byte filenames",- () => {- const f = fixture();- write(f.input, [candidate("new.py")]);- write(shard(f, 1), [candidate("preserved.py")]);- writeFileSync(join(f.directory, "é.txt"), "unrelated Unicode file");- if (process.platform === "linux")- writeFileSync(- Buffer.concat([Buffer.from(f.directory + "/"), Buffer.from([255])]),- "unrelated undecodable file",- );- const original = readFileSync(shard(f, 1));- const result = make(f);- expect(result.status).toBe(1);- expect(result.stderr).toContain("already contains shard files");- expect(readFileSync(shard(f, 1))).toEqual(original);- rmSync(shard(f, 1));- expect(make(f).status).toBe(0);- expect(read(shard(f, 1))).toEqual([candidate("new.py")]);- },- );-- test("validates one completed shard independently and restores authoritative merge order", () => {- const f = fixture();- const rows = Array.from({ length: 7 }, (_, index) =>- candidate(`src/file_${index}.py`),- );- write(f.input, rows);- expect(make(f, ["--max-rows", "5"]).status).toBe(0);- write(shard(f, 1, true), read(shard(f, 1)).reverse().map(ranked));- const checked = validate(f);- expect(checked.status).toBe(0);- expect(checked.stdout).toBe(- `Validated 5 ranking rows in ${shard(f, 1, true)}${newline}`,- );- expect(existsSync(shard(f, 2, true))).toBe(false);- complete(f);- const result = merge(f);- expect(result.status).toBe(0);- expect(result.stdout).toBe(- `Merged 7 ranking rows into ${f.output}${newline}`,- );- expect(read(f.output)).toEqual(rows.map(ranked));- });-- test("creates zero shards and merges an empty input while ignoring unrelated files", () => {- const f = fixture();- write(f.input, []);- mkdirSync(f.directory);- writeFileSync(join(f.directory, "notes.txt"), "retain notes");- expect(make(f).stdout).toBe(- `Wrote 0 rank shards to ${f.directory}${newline}`,- );- writeFileSync(f.output, "previous contents");- expect(merge(f).stdout).toBe(- `Merged 0 ranking rows into ${f.output}${newline}`,- );- expect(readFileSync(f.output)).toHaveLength(0);- expect(readFileSync(join(f.directory, "notes.txt"), "utf8")).toBe(- "retain notes",- );- write(shard(f, 1), []);- write(shard(f, 1, true), []);- expect(validate(f).status).toBe(0);- });-- test.each(["1", "+20", "20"])("accepts decimal shard size %s", (size) => {- const f = fixture();- write(f.input, [candidate("a.py"), candidate("b.py")]);- expect(make(f, [`--max-rows=${size}`]).status).toBe(0);- expect(readdirSync(f.directory)).toHaveLength(size === "1" ? 2 : 1);- });-- test.each(["0", "-1"])(- "rejects nonpositive shard size %s before input access",- (size) => {- const f = fixture();- const result = make(f, [`--max-rows=${size}`]);- expect(result.status).toBe(1);- expect(result.stderr).toBe(`--max-rows must be at least 1${newline}`);- expect(existsSync(f.directory)).toBe(false);- },- );-- test("uses standard named options, last repeated values, and argument failures", () => {- const f = fixture();- write(f.input, [candidate("a.py"), candidate("b.py")]);- expect(make(f, ["--max-rows", "1", "--max-rows=2"]).status).toBe(0);- expect(readdirSync(f.directory)).toHaveLength(1);- for (const args of [- ["--max-rows", "1.0"],- ["--max-rows", "1__0"],- ["--max-rows", "\u001c20"],- ["--max-rows"],- ["--unknown"],- ["extra"],- ]) {- expect(make(f, args).status).toBe(2);- }- for (const command of [- "make-rank-shards",- "validate-rank-shard",- "merge-rank-outputs",- ]) {- expect(run(f, command, []).status).toBe(2);- expect(run(f, command, ["--help"]).status).toBe(0);- }- expect(make(f, ["--help"]).stdout).toContain("Defaults to 150");- complete(f);- expect(merge(f, ["--out", f.output]).status).toBe(0);- });-- test("preserves JSON fields and Unicode contents", () => {- const f = fixture();- writeFileSync(- f.input,- '{"preview":"é\\n","path":"old","path":"𐀀.py","area":"\\udcff"}\r',- );- expect(make(f).status).toBe(0);- expect(read(shard(f, 1))).toEqual([- { preview: "é\n", path: "𐀀.py", area: "\udcff" },- ]);- writeFileSync(- shard(f, 1, true),- '{"reason":"é","include":false,"score":5,"area":"\\udcff","path":"𐀀.py"}',- );- expect(merge(f).status).toBe(0);- expect(read(f.output)).toEqual([- { reason: "é", include: false, score: 5, area: "\udcff", path: "𐀀.py" },- ]);- });-- test.each([- ["{not json}\n", "invalid JSON"],- [- '{"path":"a.py","area":"src","score":10,"include":true}',- 'missing fields ["reason"]',- ],- [- JSON.stringify({ ...ranked(candidate("a.py")), score: true }),- "score must be an integer",- ],- [- '{"path":"a.py","area":"src","score":5.5,"include":true,"reason":"x"}',- "score must be an integer",- ],- [- JSON.stringify({ ...ranked(candidate("a.py")), score: 11 }),- "score must be from 1 through 10",- ],- [- JSON.stringify({ ...ranked(candidate("a.py")), include: "true" }),- "include must be a boolean",- ],- [- JSON.stringify({ ...ranked(candidate("a.py")), reason: "\t" }),- "reason must be a non-empty string",- ],- [- JSON.stringify(ranked(candidate("b.py"))),- "paths do not match its input shard",- ],- [- JSON.stringify(ranked(candidate("a.py", "other"))),- "area does not match rank input",- ],- ["", "paths do not match its input shard"],- ["\n", "blank JSONL rows are not allowed"],- [- JSON.stringify({ ...ranked(candidate("a.py")), preview: "extra" }),- 'unexpected fields ["preview"]',- ],- ])(- "rejects invalid worker output %j without overwriting the merged file",- (data, message) => {- const f = fixture();- write(f.input, [candidate("a.py")]);- expect(make(f).status).toBe(0);- writeFileSync(shard(f, 1, true), data!);- writeFileSync(f.output, "preserve existing output");- for (const result of [validate(f), merge(f)]) {- expect(result.status).toBe(1);- expect(result.stderr).toContain(message!);- }- expect(readFileSync(f.output, "utf8")).toBe("preserve existing output");- },- );-- test("rejects missing, unexpected, and duplicate worker outputs", () => {- const f = fixture();- for (const result of [make(f), merge(f)]) {- expect(result.status).toBe(1);- expect(result.stderr).toBe(`Rank input missing: ${f.input}${newline}`);- }- const missingInput = validate(f);- expect(missingInput.status).toBe(1);- expect(missingInput.stderr).toBe(- `Rank input shard missing: ${shard(f, 1)}${newline}`,- );- write(f.input, [candidate("a.py")]);- expect(make(f).status).toBe(0);- const missingOutput = validate(f);- expect(missingOutput.status).toBe(1);- expect(missingOutput.stderr).toBe(- `Rank output shard missing: ${shard(f, 1, true)}${newline}`,- );- write(shard(f, 2, true), []);- expect(merge(f).stderr).toContain(- 'missing output shards ["rank-shard-0001.output.jsonl"]; unexpected output shards ["rank-shard-0002.output.jsonl"]',- );- rmSync(shard(f, 2, true));- const row = ranked(candidate("a.py"));- write(shard(f, 1, true), [row, row]);- expect(validate(f).stderr).toContain(- 'Rank output shard rank-shard-0001.output.jsonl contains duplicate paths: ["a.py"]',- );- expect(merge(f).stderr).toContain("duplicate paths");- });-- test.skipIf(process.platform === "win32")(- "escapes terminal controls in native shard output errors",- () => {- const f = fixture();- write(f.input, [candidate("a.py")]);- f.directory = join(f.root, "shards-\x1b[2J");- writeFileSync(f.directory, "existing file");- const result = make(f);- expect(result.status).toBe(1);- expect(result.stderr).toMatch(/\\(?:x1b|u001b)\[2J/);- expect(result.stderr).not.toContain("\x1b");- expect(readFileSync(f.directory, "utf8")).toBe("existing file");- },- );-- test.each([- "rank-shard-x.input.jsonl",- "rank-shard-001.input.jsonl",- "rank-shard-0000.input.jsonl",- "rank-shard-00001.input.jsonl",- "rank-shard-0002.input.jsonl",- ])("rejects noncanonical shard name %s", (name) => {- const f = fixture();- write(f.input, [candidate("a.py")]);- expect(make(f).status).toBe(0);- renameSync(shard(f, 1), join(f.directory, name));- const result = merge(f);- expect(result.status).toBe(1);- expect(result.stderr).toContain(name);- expect(existsSync(f.output)).toBe(false);- });-- test("rejects gaps in shard names above four digits", () => {- const f = fixture();- write(f.input, []);- write(join(f.directory, "rank-shard-9999.input.jsonl"), []);- write(join(f.directory, "rank-shard-10000.input.jsonl"), []);- const result = merge(f);- expect(result.status).toBe(1);- expect(result.stderr).toContain("contiguous canonical names");- expect(existsSync(f.output)).toBe(false);- });-- test.each([- "RANK-SHARD-0001.INPUT.JSONL",- "rank-shard-0001.İnput.jsonl",- "rank-shard-0001.ınput.jsonl",- "rank-shard-0001.input.jſonl",- "ranK-shard-0001.input.jsonl",- ])(- "matches shard globs using the platform's filename case rules: %s",- (name) => {- const f = fixture();- write(f.input, []);- write(join(f.directory, name), []);- const creation = make(f);- const result = merge(f);- if (process.platform === "win32") {- expect(creation.status).toBe(1);- expect(creation.stderr).toContain("already contains shard files");- expect(result.status).toBe(1);- expect(result.stderr).toContain("contiguous canonical names");- } else {- expect(creation.status).toBe(0);- expect(result.status).toBe(0);- expect(readFileSync(f.output)).toHaveLength(0);- }- },- );-- test.each(["order", "preview", "duplicate across shards"])(- "rejects an invalid authoritative partition: %s",- (change) => {- const f = fixture();- const rows = [candidate("a.py"), candidate("b.py")];- write(f.input, rows);- expect(make(f, ["--max-rows", "1"]).status).toBe(0);- if (change === "order") write(f.input, [...rows].reverse());- if (change === "preview")- write(shard(f, 1), [{ ...rows[0], preview: "changed" }]);- if (change === "duplicate across shards") write(shard(f, 2), [rows[0]!]);- complete(f);- const result = merge(f);- expect(result.status).toBe(1);- expect(result.stderr).toContain(- change === "duplicate across shards"- ? "Rank outputs contain duplicate path: a.py"- : "do not exactly partition the authoritative rank input",- );- expect(existsSync(f.output)).toBe(false);- },- );-- test("rejects duplicate or malformed authoritative rows before creating a shard directory", () => {- const f = fixture();- for (const data of [- JSON.stringify(candidate("a.py")) +- "\n" +- JSON.stringify(candidate("a.py")),- "{}",- "\u001c\n",- "\ufeff{}",- Buffer.from([255]),- ]) {- writeFileSync(f.input, data);- expect(make(f).status).toBe(1);- expect(merge(f).status).toBe(1);- expect(existsSync(f.directory)).toBe(false);- }- });-- test("treats a matching directory as an existing shard and reports a missing shard directory", () => {- const f = fixture();- write(f.input, []);- expect(merge(f).stderr).toContain("Rank shard directory missing");- mkdirSync(shard(f, 1, true), { recursive: true });- expect(make(f).stderr).toContain("already contains shard files");- });-- test.skipIf(process.platform === "win32")(- "preserves raw shard names without requiring entry metadata",- () => {- const f = fixture();- write(f.input, []);- mkdirSync(f.directory);- const name = Buffer.concat([- Buffer.from("rank-shard-"),- process.platform === "darwin" ? Buffer.from("é") : Buffer.from([255]),- Buffer.from(".input.jsonl"),- ]);- writeFileSync(Buffer.concat([Buffer.from(f.directory + "/"), name]), "");- chmodSync(f.directory, 0o400);- try {- const creation = make(f);- expect(creation.status).toBe(1);- expect(creation.stderr).toContain("already contains shard files");- const result = merge(f);- expect(result.status).toBe(1);- expect(result.stderr).toContain("contiguous canonical names");- expect(result.stderr).toContain(- process.platform === "darwin"- ? "rank-shard-é.input.jsonl"- : "rank-shard-\\udcff.input.jsonl",- );- expect(existsSync(f.output)).toBe(false);- } finally {- chmodSync(f.directory, 0o700);- }- },- );-- test("uses literal dash paths, expands homes, and accepts output/input aliasing", () => {- const f = fixture();- write(join(f.root, "-"), [candidate("a.py")]);- const env = { ...process.env, HOME: f.root, USERPROFILE: f.root };- const result = run(- f,- "make-rank-shards",- ["--rank-input", "-", "--out-dir", "~/shards"],- env,- );- expect(result.status).toBe(0);- complete(f);- const merged = merge({ ...f, input: "-", output: "-" });- expect(merged.status).toBe(0);- expect(merged.stdout).toBe(`Merged 1 ranking rows into -${newline}`);- expect(read(join(f.root, "-"))).toEqual([ranked(candidate("a.py"))]);- });-- test.skipIf(process.platform === "win32")(- "keeps symlink/.. traversal and existing output permissions",- () => {- const f = fixture();- const outside = fixture();- mkdirSync(join(outside.root, "nested"));- symlinkSync(join(outside.root, "nested"), join(f.root, "link"));- const directory = f.root + "/link/../shards";- write(f.input, [candidate("a.py")]);- expect(make({ ...f, directory, input: f.input + "/." }).status).toBe(0);- expect(- existsSync(join(outside.root, "shards", "rank-shard-0001.input.jsonl")),- ).toBe(true);- expect(existsSync(f.directory)).toBe(false);- complete({ ...f, directory });- const target = join(f.root, "target.jsonl");- writeFileSync(target, "existing", { mode: 0o640 });- symlinkSync(target, f.output);- expect(merge({ ...f, directory, output: f.output + "/./" }).status).toBe(- 0,- );- expect(statSync(target).mode & 0o777).toBe(0o640);- expect(read(target)).toEqual([ranked(candidate("a.py"))]);- },- );-- test.skipIf(process.platform === "win32")(- "launcher preserves POSIX shard directory bytes",- () => {- const f = fixture();- const pathBytes =- process.platform === "darwin" ? Buffer.from("é") : Buffer.from([255]);- const suffix = Array.from(- pathBytes,- (byte) => `\\${byte.toString(8).padStart(3, "0")}`,- ).join("");- write(f.input, [candidate("a.py")]);- const launcher = join(- PLUGIN_ROOT,- "scripts",- "launch_codex_security_mcp",- );- const result = spawnSync(- "sh",- [- "-c",- `exec "$1" --helper make-rank-shards --rank-input "$2/input.jsonl" --out-dir "$2/shards-$(printf '${suffix}')"`,- "sh",- launcher,- f.root,- ],- { env: { ...process.env, CODEX_MCP_NODE_PATH: node } },- );- expect(result.status).toBe(0);- const directory = Buffer.concat([- Buffer.from(f.root + "/shards-"),- pathBytes,- ]);- const input = Buffer.concat([- directory,- Buffer.from("/rank-shard-0001.input.jsonl"),- ]);- const output = Buffer.concat([- directory,- Buffer.from("/rank-shard-0001.output.jsonl"),- ]);- writeFileSync(output, JSON.stringify(ranked(candidate("a.py"))) + "\n");- expect(JSON.parse(readFileSync(input, "utf8"))).toEqual(- candidate("a.py"),- );- const merged = spawnSync(- "sh",- [- "-c",- `exec "$1" --helper merge-rank-outputs --rank-input "$2/input.jsonl" --shard-dir "$2/shards-$(printf '${suffix}')" --out "$2/output.jsonl"`,- "sh",- launcher,- f.root,- ],- { env: { ...process.env, CODEX_MCP_NODE_PATH: node } },- );- expect(merged.status).toBe(0);- expect(read(f.output)).toEqual([ranked(candidate("a.py"))]);- },- );-});diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.tsindex 3337c4a54..d723426aa 100644--- a/sdk/typescript/tests-ts/runtime.test.ts+++ b/sdk/typescript/tests-ts/runtime.test.ts@@ -491,18 +491,20 @@ describe("plugin runtime preparation", () => {expect(inventory.status).toBe(2);expect(inventory.stderr).toContain("NTFS alternate data streams");- const rankInput = runNodePython(python!, [+ const scopes = join(root, "scopes.json");+ await writeFile(scopes, JSON.stringify(["source.ts:synthetic-stream"]));+ const scopeInput = runNodePython(python!, [join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),- "make-repo-rank-input",+ "make-repo-scope-input","--repo",repository,- "--scope",- "source.ts:synthetic-stream",+ "--scopes-file",+ scopes,"--out",- join(root, "rank-input.jsonl"),+ join(root, "scope-input.jsonl"),]);- expect(rankInput.status).toBe(1);- expect(rankInput.stderr).toContain("NTFS alternate data stream");+ expect(scopeInput.status).toBe(1);+ expect(scopeInput.stderr).toContain("NTFS alternate data stream");},);@@ -6276,14 +6278,18 @@ describe("runtime directories and plugin Python boundary", () => {const python = Bun.which("python3") ?? Bun.which("python");expect(python).not.toBeNull();+ const scopes = join(root, "scopes.json");+ await writeFile(scopes, JSON.stringify(["."]));const result = spawnSync(python!,["-B",join(PLUGIN_ROOT, "scripts", "generate_rank_input.py"),- "make-repo-rank-input",+ "make-repo-scope-input","--repo",repository,+ "--scopes-file",+ scopes,"--out",output,],