OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(windows): preserve scoped inventories after case-only renames (#633)
faizan-oaic795ab4a7d02
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.pyindex a869ebb22..47fcde677 100644--- a/plugins/codex-security/scripts/workbench_target.py+++ b/plugins/codex-security/scripts/workbench_target.py@@ -374,21 +374,55 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:if repository_root is None:return Nonerepository, pathspec = git_worktree_context(target)+ scope = repository / pathspec+ scope_depth = len(Path(pathspec).parts)+ matching_prefixes: dict[str, bool] = {}+ listing_args: list[str] = []+ inventory_pathspec = pathspec+ if scope_depth:+ if any(+ not character.isascii() and character.lower() != character.upper()+ for character in pathspec+ ):+ # Git's icase pathspecs do not cover Unicode case aliases.+ inventory_pathspec = "."+ else:+ listing_args.append("--no-literal-pathspecs")+ inventory_pathspec = f":(icase,literal){pathspec}"listed = git_bytes(repository,+ *listing_args,"ls-files","--cached","--others","--exclude-standard","-z","--",- pathspec,+ inventory_pathspec,)if listed is None:raise SystemExit("Could not inspect files in the selected Git working tree.")paths: list[Path] = []for raw_path in (raw_path for raw_path in listed.split(b"\0") if raw_path):- path = repository / os.fsdecode(raw_path)+ relative = Path(os.fsdecode(raw_path))+ path = repository / relative+ if scope_depth:+ if len(relative.parts) <= scope_depth:+ continue+ # Git's index spelling can differ after a case-only directory rename.+ # Compare each scope-depth prefix once, without following symlink leaves.+ prefix = repository.joinpath(*relative.parts[:scope_depth])+ key = str(prefix)+ if key not in matching_prefixes:+ try:+ matching_prefixes[key] = prefix.samefile(scope)+ except (FileNotFoundError, NotADirectoryError):+ matching_prefixes[key] = False+ # realpath spelling is not a filesystem identity on case-insensitive+ # POSIX volumes; WindowsPath equality also folds distinct names.+ if not matching_prefixes[key]:+ continue+ path = scope.joinpath(*relative.parts[scope_depth:])try:metadata = path.lstat()except FileNotFoundError:@@ -411,7 +445,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:for nested_path in path.rglob("*")if ".git" not in nested_path.relative_to(path).parts)- return sorted(set(paths))+ return sorted({str(path): path for path in paths}.values(), key=str)def source_directory_snapshot_paths(target: Path) -> list[Path]:diff --git a/sdk/typescript/src/component-plan.ts b/sdk/typescript/src/component-plan.tsindex 27c619475..c75380bbc 100644--- a/sdk/typescript/src/component-plan.ts+++ b/sdk/typescript/src/component-plan.ts@@ -193,9 +193,11 @@ async function inventoryFiles(throw error;},);- if (metadata?.isFile()) files.push(path);+ if (metadata?.isFile()) files.push(join(repository, path));}- return files.sort();+ return files.length === 0+ ? []+ : [...(await normalizeTarget(repository, files, signal)).paths].sort();}const files: string[] = [];const pending = [""];diff --git a/sdk/typescript/src/targets.ts b/sdk/typescript/src/targets.tsindex 032cf9317..237bbd3cc 100644--- a/sdk/typescript/src/targets.ts+++ b/sdk/typescript/src/targets.ts@@ -251,7 +251,7 @@ export async function normalizeTarget();}- const paths: string[] = [];+ const paths = new Set<string>();for (const value of target) {throwIfAborted(signal);if (typeof value !== "string") {@@ -298,11 +298,9 @@ export async function normalizeTarget();}const normalized = relativePath.split(sep).join("/") || ".";- if (!paths.includes(normalized)) {- paths.push(normalized);- }+ paths.add(normalized);}- return { kind: "paths", paths };+ return { kind: "paths", paths: [...paths] };}export async function validateCommittedDiffCheckout(diff --git a/sdk/typescript/tests-ts/component-scan.test.ts b/sdk/typescript/tests-ts/component-scan.test.tsindex 30e013b61..bcb672760 100644--- a/sdk/typescript/tests-ts/component-scan.test.ts+++ b/sdk/typescript/tests-ts/component-scan.test.ts@@ -3,13 +3,16 @@ import {mkdir,mkdtemp,readFile,+ readdir,realpath,+ rename,rm,+ stat,symlink,writeFile,} from "node:fs/promises";import { tmpdir } from "node:os";-import { dirname, join } from "node:path";+import { dirname, join, relative, sep } from "node:path";import { afterEach, expect, test } from "bun:test";import type { ScanOptions } from "../src/api.js";import { main } from "../src/cli.js";@@ -26,6 +29,7 @@ import {} from "../src/component-scan.js";import type { Finding, SeverityLevel } from "../src/models.js";import { ScanResult } from "../src/result.js";+import { normalizeTarget } from "../src/targets.js";import {matchScanFindings,type ScanComparisonInput,@@ -39,6 +43,7 @@ import {fakeResult,FakeSignals,} from "./cli-fixtures.js";+import { PLUGIN_ROOT } from "./plugin-root.js";const temporary: string[] = [];const components: ComponentPlan["components"] = [@@ -182,6 +187,55 @@ function fakeCodex(};}+async function scopedInventory(paths: Fixture, scope: string) {+ const python =+ process.env["PYTHON"] ?? Bun.which("python3") ?? Bun.which("python");+ expect(python).not.toBeNull();+ const scopesFile = join(paths.root, "scopes.json");+ const output = join(paths.root, "scoped-source-input.jsonl");+ await writeFile(scopesFile, JSON.stringify([scope]));+ const stdout = execFileSync(+ python!,+ [+ "-I",+ "-B",+ "-c",+ [+ "import json, sys",+ "from argparse import Namespace",+ "from pathlib import Path",+ "sys.path.insert(0, sys.argv[1])",+ "import workbench_target as target",+ "from generate_rank_input import make_repo_scope_input",+ "queries = []",+ "git_bytes = target.git_bytes",+ "def record_query(repository, *args, **kwargs):",+ " data = git_bytes(repository, *args, **kwargs)",+ " if 'ls-files' in args:",+ " queries.append({'pathspec': args[-1], 'count': len([path for path in (data or b'').split(b'\\0') if path])})",+ " return data",+ "target.git_bytes = record_query",+ "repo, scope, scopes, output = sys.argv[2:]",+ "make_repo_scope_input(Namespace(repo=repo, scopes_file=scopes, out=output))",+ "rows = [json.loads(line)['path'] for line in Path(output).read_text().splitlines()]",+ "count = target.directory_snapshot_regular_file_count((Path(repo) / scope).resolve())",+ "print(json.dumps({'paths': rows, 'count': count, 'queries': queries}))",+ ].join("\n"),+ join(PLUGIN_ROOT, "scripts"),+ paths.repository,+ scope,+ scopesFile,+ output,+ ],+ { encoding: "utf8", stdio: "pipe" },+ );+ return JSON.parse(stdout.trim().split("\n").at(-1)!) as {+ paths: string[];+ count: number;+ queries: Array<{ pathspec: string; count: number }>;+ };+}+function matcher(response: (input: ScanComparisonInput,@@ -732,6 +786,16 @@ test("plans from a Git inventory without tools or ignored files", async () => {paths: [".gitignore", "apps/web", "package.json", "shared"],},]);+ await mkdir(join(paths.repository, "~"));+ await writeFile(join(paths.repository, "~", "app.ts"), "export {};\n");+ const wholeRepository = {+ components: [{ name: "Repository", paths: ["."] }],+ };+ expect(+ await planComponents(paths.repository, {+ codex: fakeCodex(() => wholeRepository),+ }),+ ).toEqual(wholeRepository);for (const path of ["ignored", "ignored/secret.txt"]) {const proposed = { components: [{ name: "Ignored", paths: [path] }] };await expect(@@ -743,6 +807,242 @@ test("plans from a Git inventory without tools or ignored files", async () => {}});+test("keeps scoped inventories and plans aligned after a case-only Git rename", async () => {+ const paths = await fixture();+ const source = "src";+ const uppercase = source.toUpperCase();+ const git = (...args: string[]) =>+ execFileSync(+ "git",+ [+ "-C",+ paths.repository,+ "-c",+ "user.name=Test",+ "-c",+ "[email protected]",+ "-c",+ "commit.gpgSign=false",+ "-c",+ "core.hooksPath=" + join(paths.root, "hooks"),+ ...args,+ ],+ {+ encoding: "utf8",+ stdio: "pipe",+ env: {+ ...process.env,+ GIT_CONFIG_NOSYSTEM: "1",+ GIT_CONFIG_GLOBAL: join(paths.root, "gitconfig"),+ },+ },+ ).trim();+ await mkdir(join(paths.repository, source, "nested"), { recursive: true });+ await writeFile(join(paths.repository, source, "app.ts"), "export {};\n");+ await writeFile(+ join(paths.repository, source, "nested", "util.ts"),+ "export {};\n",+ );+ await mkdir(join(paths.repository, source, "build"));+ await writeFile(+ join(paths.repository, source, "build", "tracked.ts"),+ "export {};\n",+ );+ await writeFile(join(paths.repository, ".gitignore"), "build/\n");+ git("init", "-q", "-b", "main");+ git("add", ".");+ git("add", "--force", source + "/build/tracked.ts");+ git("commit", "-qm", "Initial fixture");+ const ordinaryPaths = [+ source + "/app.ts",+ source + "/build/tracked.ts",+ source + "/nested/util.ts",+ ];+ expect(await scopedInventory(paths, source)).toEqual({+ paths: ordinaryPaths,+ count: 3,+ queries: [+ { pathspec: ":(icase,literal)" + source, count: 3 },+ { pathspec: ":(icase,literal)" + source, count: 3 },+ ],+ });+ git("switch", "-c", "case-rename");+ git("mv", source, "renaming");+ git("mv", "renaming", uppercase);+ git("commit", "-qm", "Rename source directory");+ await writeFile(+ join(paths.repository, uppercase, "build", "cache.tmp"),+ "ignored build output\n",+ );+ git("switch", "main");+ expect(git("status", "--porcelain")).toBe("");+ expect(+ git("ls-files", "-z", "--", source).split("\0").filter(Boolean),+ ).toEqual(ordinaryPaths);+ const entries = await readdir(paths.repository);+ expect(entries).toContain(uppercase);++ const inventory = async (scope: string) => {+ const { queries, ...selected } = await scopedInventory(paths, scope);+ const pathspec = scope === "." ? "." : ":(icase,literal)" + scope;+ expect(queries.map((query) => query.pathspec)).toEqual([+ pathspec,+ pathspec,+ ]);+ if (scope === ".") {+ selected.paths = (+ await Promise.all(+ selected.paths.map(async (path) =>+ relative(+ paths.repository,+ await realpath(join(paths.repository, path)),+ )+ .split(sep)+ .join("/"),+ ),+ )+ ).sort();+ }+ return selected;+ };++ const target = await normalizeTarget(paths.repository, [source]);+ const scope = target.paths[0]!;+ const expectedInventory = {+ paths: [+ scope + "/app.ts",+ scope + "/build/tracked.ts",+ scope + "/nested/util.ts",+ ],+ count: 3,+ };+ expect(await inventory(scope)).toEqual(expectedInventory);+ await writeFile(+ join(paths.repository, scope, "untracked.ts"),+ "export {};\n",+ );+ const mixedInventory = {+ paths: [...expectedInventory.paths, scope + "/untracked.ts"].sort(),+ count: 4,+ };+ expect(await scopedInventory(paths, scope)).toEqual({+ ...mixedInventory,+ queries: [+ { pathspec: ":(icase,literal)" + scope, count: 4 },+ { pathspec: ":(icase,literal)" + scope, count: 4 },+ ],+ });+ const repositoryInventory = {+ paths: [+ ".gitignore",+ "apps/api/app.ts",+ "apps/web/app.ts",+ "package.json",+ "shared/util.ts",+ ...mixedInventory.paths,+ ].sort(),+ count: 9,+ };+ expect(await inventory(".")).toEqual(repositoryInventory);+ const proposed = { components: [{ name: "Source", paths: [source] }] };+ const other = {+ name: "Other files",+ paths: [".gitignore", "apps", "package.json", "shared"],+ };+ expect(+ await planComponents(paths.repository, {+ codex: fakeCodex(() => proposed),+ }),+ ).toEqual({+ components: [{ name: "Source", paths: [scope] }, other],+ });++ if (entries.includes(source)) {+ await writeFile(+ join(paths.repository, uppercase, "app.ts"),+ "export {};\n",+ );+ expect(await inventory(source)).toEqual(mixedInventory);+ expect(await inventory(uppercase)).toEqual({+ paths: [uppercase + "/app.ts"],+ count: 1,+ });+ expect(await inventory(".")).toEqual({+ paths: [...repositoryInventory.paths, uppercase + "/app.ts"].sort(),+ count: 10,+ });+ const separate = { name: "Separate source", paths: [uppercase] };+ expect(+ await planComponents(paths.repository, {+ codex: fakeCodex(() => ({+ components: [...proposed.components, separate],+ })),+ }),+ ).toEqual({ components: [...proposed.components, separate, other] });+ }+});++test("retains tracked Unicode aliases when scoped Git matching is incomplete", async () => {+ const paths = await fixture();+ const source = "sourcé";+ const uppercase = source.toUpperCase();+ await mkdir(join(paths.repository, source, "build"), { recursive: true });+ await writeFile(join(paths.repository, source, "app.ts"), "export {};\n");+ await writeFile(+ join(paths.repository, source, "build", "tracked.ts"),+ "export {};\n",+ );+ await writeFile(join(paths.repository, ".gitignore"), "build/\n");+ const nonCased = "项目api";+ await mkdir(join(paths.repository, nonCased));+ await writeFile(join(paths.repository, nonCased, "app.ts"), "export {};\n");+ execFileSync("git", ["-C", paths.repository, "init", "-q"]);+ execFileSync("git", ["-C", paths.repository, "add", "--force", "."]);+ expect((await scopedInventory(paths, nonCased)).queries).toEqual([+ { pathspec: ":(icase,literal)" + nonCased, count: 1 },+ { pathspec: ":(icase,literal)" + nonCased, count: 1 },+ ]);+ await rename(+ join(paths.repository, source),+ join(paths.repository, "renaming"),+ );+ await rename(+ join(paths.repository, "renaming"),+ join(paths.repository, uppercase),+ );+ const aliases =+ (await realpath(join(paths.repository, source)).catch(+ (error: NodeJS.ErrnoException) => {+ if (error.code === "ENOENT") return null;+ throw error;+ },+ )) !== null;+ await writeFile(+ join(paths.repository, uppercase, "untracked.ts"),+ "export {};\n",+ );+ const selected = [+ uppercase + "/app.ts",+ uppercase + "/untracked.ts",+ ...(aliases ? [uppercase + "/build/tracked.ts"] : []),+ ].sort();+ const inventory = await scopedInventory(paths, uppercase);+ const identities = async (files: string[]) =>+ (+ await Promise.all(+ files.map(async (file) => {+ const { dev, ino } = await stat(join(paths.repository, file), {+ bigint: true,+ });+ return `${dev}:${ino}`;+ }),+ )+ ).sort();+ expect(await identities(inventory.paths)).toEqual(await identities(selected));+ expect(inventory.count).toBe(selected.length);+ expect(inventory.queries.map((query) => query.pathspec)).toEqual([".", "."]);+});+test("plans plain directories and rejects unsafe or overlapping model scopes", async () => {const paths = await fixture();const proposed = { components: [{ name: "Apps", paths: ["apps"] }] };diff --git a/sdk/typescript/tests-ts/targets.test.ts b/sdk/typescript/tests-ts/targets.test.tsindex 8a670666d..227b5217b 100644--- a/sdk/typescript/tests-ts/targets.test.ts+++ b/sdk/typescript/tests-ts/targets.test.ts@@ -119,7 +119,12 @@ describe("scan target normalization", () => {paths: [],});expect(- await normalizeTarget(repo, ["src", join(repo, "src", "app.ts")]),+ await normalizeTarget(repo, [+ "src",+ join(repo, "src", "app.ts"),+ join(repo, "src"),+ "src/app.ts",+ ]),).toEqual({kind: "paths",paths: ["src", "src/app.ts"],