OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
refactor(imports): share local Git metadata discovery (#2194)
Michael D'Angelofe1f1dc63eed
diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjsindex ad94df7e8..70f5b8f65 100644--- a/sdk/typescript/scripts/package-dist-files.mjs+++ b/sdk/typescript/scripts/package-dist-files.mjs@@ -76,6 +76,7 @@ const distModules = ["finding-catalogue","findings-import","git-remote",+ "local-git-metadata","github","github-advisories","github-dependabot-findings",diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.tsindex 8be8c7bd5..7f648ae65 100644--- a/sdk/typescript/src/external-findings-local.ts+++ b/sdk/typescript/src/external-findings-local.ts@@ -1,7 +1,4 @@-import { lstat, readFile, realpath, stat } from "node:fs/promises";-import { dirname, isAbsolute, join, resolve, sep } from "node:path";import { isNumberObject } from "node:util/types";-import { environmentEntry } from "./auth.js";import { CodexSecurityError } from "./errors.js";import {forwardExternalImportSetting,@@ -23,6 +20,10 @@ import type {Provider,} from "./external-import-models.js";import { localFindingRepositories } from "./external-findings-read.js";+import {+ hasGitMetadata,+ localGitProtectionRoots,+} from "./local-git-metadata.js";import {blackDuckReportRepositories,projectedBlackDuckRepositoryUrl,@@ -53,11 +54,8 @@ import {import {abortable,enclosingGitWorktreeRoot,- gitMarkerRoot,gitOutput,- gitProtectionRoots,normalizeRepository,- nullIfMissingPath,} from "./targets.js";/** A scanner source record, independent of a Codex assessment or completed scan. */@@ -405,194 +403,6 @@ function sameScannerRepositories(return true;}-async function bareMetadataRoots(- repository: string,- signal?: AbortSignal,-): Promise<string[]> {- const roots: string[] = [];- let directory = repository;- for (;;) {- signal?.throwIfAborted();- const present = async (name: string) =>- (await abortable(- () => lstat(join(directory, name)).catch(nullIfMissingPath),- signal,- )) !== null;- // Git also discovers bare repositories and linked metadata without .git.- const head = await abortable(- () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),- signal,- );- if (- (head?.isFile() || head?.isSymbolicLink()) &&- ((await present("commondir")) ||- ((await present("objects")) && (await present("refs"))))- )- roots.push(directory);- const parent = dirname(directory);- if (parent === directory) return roots;- directory = parent;- }-}--async function metadataPointer(- path: string,- prefix: string,- context: { directory: string; environment: NodeJS.ProcessEnv },- signal?: AbortSignal,-): Promise<string | undefined> {- const metadata = await abortable(- () => stat(path).catch(nullIfMissingPath),- signal,- );- if (!metadata?.isFile()) return undefined;- const text = await abortable(() => readFile(path, "utf8"), signal);- if (!text.startsWith(prefix) || (prefix === "" && text.length === 0))- return undefined;- const payload = text.slice(prefix.length).replace(/[\r\n]+$/u, "");- if (prefix !== "" && payload.length === 0) return undefined;- // Git checks emptiness before consuming the path as a NUL-terminated string.- const value = payload.split("\0", 1)[0]!;- // Preserve filesystem traversal through links before '..'.- let target = isAbsolute(value) ? value : `${dirname(path)}${sep}${value}`;- if (process.platform === "win32") {- const drive = /^[A-Za-z]:(?![\\/])/u.exec(value)?.[0].toUpperCase();- if (drive !== undefined) {- const directory =- context.directory.slice(0, 2).toUpperCase() === drive- ? context.directory- : environmentEntry(context.environment, `=${drive}`) || `${drive}\\`;- target = resolve(directory, value.slice(2));- } else if (isAbsolute(value)) target = resolve(context.directory, value);- }- return (- (await abortable(- () => realpath(target).catch(nullIfMissingPath),- signal,- )) ?? undefined- );-}--async function associatedMetadataRoots(- directory: string,- context: { directory: string; environment: NodeJS.ProcessEnv },- signal?: AbortSignal,- configurationDirectory?: string,-): Promise<string[]> {- const head = await abortable(- () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),- signal,- );- if (!head || (!head.isFile() && !head.isSymbolicLink())) return [];- const commonMarker = join(directory, "commondir");- const hasCommon = await abortable(- () => lstat(commonMarker).catch(nullIfMissingPath),- signal,- );- const common = hasCommon- ? await metadataPointer(commonMarker, "", context, signal)- : undefined;- if (hasCommon && common === undefined) return [];- const storage = common ?? directory;- for (const name of ["objects", "refs"]) {- const metadata = await abortable(- () => stat(join(storage, name)).catch(nullIfMissingPath),- signal,- );- if (!metadata?.isDirectory()) return [];- }- const roots = common === undefined ? [directory] : [directory, common];- if (- process.platform === "win32" &&- configurationDirectory !== undefined &&- configurationDirectory !== context.directory- ) {- // Git rereads commondir after changing to the discovered repository root.- roots.push(- ...(await associatedMetadataRoots(- directory,- {- ...context,- directory: await abortable(- () => canonicalConfigPath(configurationDirectory),- signal,- ),- },- signal,- )),- );- }- return roots;-}--export async function localGitProtectionRoots(- repository: string,- environment: NodeJS.ProcessEnv,- signal?: AbortSignal,-): Promise<string[]> {- const roots = new Set(- await abortable(() => gitProtectionRoots(repository, signal), signal),- );- const context = {- directory: await abortable(() => canonicalConfigPath(repository), signal),- environment,- };- const paths = new Set([resolve(repository), context.directory]);- for (const path of paths) {- for (const root of await bareMetadataRoots(path, signal)) {- roots.add(root);- for (const associated of await associatedMetadataRoots(- root,- context,- signal,- root,- ))- roots.add(associated);- }- let directory = path;- for (;;) {- signal?.throwIfAborted();- const marker = join(directory, ".git");- const metadata = await abortable(- () => stat(marker).catch(nullIfMissingPath),- signal,- );- if (metadata !== null) {- roots.add(directory);- const gitDirectory = metadata.isDirectory()- ? await abortable(() => realpath(marker), signal)- : await metadataPointer(marker, "gitdir: ", context, signal);- if (gitDirectory !== undefined)- for (const associated of await associatedMetadataRoots(- gitDirectory,- context,- signal,- directory,- ))- roots.add(associated);- }- const parent = dirname(directory);- if (parent === directory) break;- directory = parent;- }- }- for (const root of [...roots])- roots.add(await abortable(() => canonicalConfigPath(root), signal));- return [...roots];-}--async function hasGitMetadata(- repository: string,- signal?: AbortSignal,-): Promise<boolean> {- return (- (await abortable(- () => gitMarkerRoot(repository, signal, "nearest"),- signal,- )) !== null || (await bareMetadataRoots(repository, signal)).length !== 0- );-}-/** @internal */export interface ConfiguredFetchRemote {name: string;diff --git a/sdk/typescript/src/local-git-metadata.ts b/sdk/typescript/src/local-git-metadata.tsnew file mode 100644index 000000000..bb80c834e--- /dev/null+++ b/sdk/typescript/src/local-git-metadata.ts@@ -0,0 +1,198 @@+import { lstat, readFile, realpath, stat } from "node:fs/promises";+import { dirname, isAbsolute, join, resolve, sep } from "node:path";+import { environmentEntry } from "./auth.js";+import { canonicalConfigPath } from "./runtime.js";+import {+ abortable,+ gitMarkerRoot,+ gitProtectionRoots,+ nullIfMissingPath,+} from "./targets.js";++async function bareMetadataRoots(+ repository: string,+ signal?: AbortSignal,+): Promise<string[]> {+ const roots: string[] = [];+ let directory = repository;+ for (;;) {+ signal?.throwIfAborted();+ const present = async (name: string) =>+ (await abortable(+ () => lstat(join(directory, name)).catch(nullIfMissingPath),+ signal,+ )) !== null;+ // Git also discovers bare repositories and linked metadata without .git.+ const head = await abortable(+ () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),+ signal,+ );+ if (+ (head?.isFile() || head?.isSymbolicLink()) &&+ ((await present("commondir")) ||+ ((await present("objects")) && (await present("refs"))))+ )+ roots.push(directory);+ const parent = dirname(directory);+ if (parent === directory) return roots;+ directory = parent;+ }+}++async function metadataPointer(+ path: string,+ prefix: string,+ context: { directory: string; environment: NodeJS.ProcessEnv },+ signal?: AbortSignal,+): Promise<string | undefined> {+ const metadata = await abortable(+ () => stat(path).catch(nullIfMissingPath),+ signal,+ );+ if (!metadata?.isFile()) return undefined;+ const text = await abortable(() => readFile(path, "utf8"), signal);+ if (!text.startsWith(prefix) || (prefix === "" && text.length === 0))+ return undefined;+ const payload = text.slice(prefix.length).replace(/[\r\n]+$/u, "");+ if (prefix !== "" && payload.length === 0) return undefined;+ // Git checks emptiness before consuming the path as a NUL-terminated string.+ const value = payload.split("\0", 1)[0]!;+ // Preserve filesystem traversal through links before '..'.+ let target = isAbsolute(value) ? value : `${dirname(path)}${sep}${value}`;+ if (process.platform === "win32") {+ const drive = /^[A-Za-z]:(?![\\/])/u.exec(value)?.[0].toUpperCase();+ if (drive !== undefined) {+ const directory =+ context.directory.slice(0, 2).toUpperCase() === drive+ ? context.directory+ : environmentEntry(context.environment, `=${drive}`) || `${drive}\\`;+ target = resolve(directory, value.slice(2));+ } else if (isAbsolute(value)) target = resolve(context.directory, value);+ }+ return (+ (await abortable(+ () => realpath(target).catch(nullIfMissingPath),+ signal,+ )) ?? undefined+ );+}++async function associatedMetadataRoots(+ directory: string,+ context: { directory: string; environment: NodeJS.ProcessEnv },+ signal?: AbortSignal,+ configurationDirectory?: string,+): Promise<string[]> {+ const head = await abortable(+ () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),+ signal,+ );+ if (!head || (!head.isFile() && !head.isSymbolicLink())) return [];+ const commonMarker = join(directory, "commondir");+ const hasCommon = await abortable(+ () => lstat(commonMarker).catch(nullIfMissingPath),+ signal,+ );+ const common = hasCommon+ ? await metadataPointer(commonMarker, "", context, signal)+ : undefined;+ if (hasCommon && common === undefined) return [];+ const storage = common ?? directory;+ for (const name of ["objects", "refs"]) {+ const metadata = await abortable(+ () => stat(join(storage, name)).catch(nullIfMissingPath),+ signal,+ );+ if (!metadata?.isDirectory()) return [];+ }+ const roots = common === undefined ? [directory] : [directory, common];+ if (+ process.platform === "win32" &&+ configurationDirectory !== undefined &&+ configurationDirectory !== context.directory+ ) {+ // Git rereads commondir after changing to the discovered repository root.+ roots.push(+ ...(await associatedMetadataRoots(+ directory,+ {+ ...context,+ directory: await abortable(+ () => canonicalConfigPath(configurationDirectory),+ signal,+ ),+ },+ signal,+ )),+ );+ }+ return roots;+}++export async function localGitProtectionRoots(+ repository: string,+ environment: NodeJS.ProcessEnv,+ signal?: AbortSignal,+): Promise<string[]> {+ const roots = new Set(+ await abortable(() => gitProtectionRoots(repository, signal), signal),+ );+ const context = {+ directory: await abortable(() => canonicalConfigPath(repository), signal),+ environment,+ };+ const paths = new Set([resolve(repository), context.directory]);+ for (const path of paths) {+ for (const root of await bareMetadataRoots(path, signal)) {+ roots.add(root);+ for (const associated of await associatedMetadataRoots(+ root,+ context,+ signal,+ root,+ ))+ roots.add(associated);+ }+ let directory = path;+ for (;;) {+ signal?.throwIfAborted();+ const marker = join(directory, ".git");+ const metadata = await abortable(+ () => stat(marker).catch(nullIfMissingPath),+ signal,+ );+ if (metadata !== null) {+ roots.add(directory);+ const gitDirectory = metadata.isDirectory()+ ? await abortable(() => realpath(marker), signal)+ : await metadataPointer(marker, "gitdir: ", context, signal);+ if (gitDirectory !== undefined)+ for (const associated of await associatedMetadataRoots(+ gitDirectory,+ context,+ signal,+ directory,+ ))+ roots.add(associated);+ }+ const parent = dirname(directory);+ if (parent === directory) break;+ directory = parent;+ }+ }+ for (const root of [...roots])+ roots.add(await abortable(() => canonicalConfigPath(root), signal));+ return [...roots];+}++export async function hasGitMetadata(+ repository: string,+ signal?: AbortSignal,+): Promise<boolean> {+ return (+ (await abortable(+ () => gitMarkerRoot(repository, signal, "nearest"),+ signal,+ )) !== null || (await bareMetadataRoots(repository, signal)).length !== 0+ );+}diff --git a/sdk/typescript/src/sonar-collector.ts b/sdk/typescript/src/sonar-collector.tsindex ac66a98c5..e4377d401 100644--- a/sdk/typescript/src/sonar-collector.ts+++ b/sdk/typescript/src/sonar-collector.ts@@ -6,8 +6,8 @@ import {boundSonarWorktree,sonarRepositoryForRemote,sonarCheckoutMatchesSource,- localGitProtectionRoots,} from "./external-findings-local.js";+import { localGitProtectionRoots } from "./local-git-metadata.js";import { repositoryUrlKey } from "./external-import-contract.js";import {forwardExternalImportSetting,diff --git a/sdk/typescript/src/sonar-import.ts b/sdk/typescript/src/sonar-import.tsindex 7daa8f728..a27005cbf 100644--- a/sdk/typescript/src/sonar-import.ts+++ b/sdk/typescript/src/sonar-import.ts@@ -8,9 +8,9 @@ import { requireExternalImports } from "./external-imports-feature.js";import {configuredFetchRemotes,type ConfiguredFetchRemote,- localGitProtectionRoots,prepareLocalImport,} from "./external-findings-local.js";+import { localGitProtectionRoots } from "./local-git-metadata.js";import {ExternalPublicationError,externalPublicationStateDirectory,