Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(imports): share local Git metadata discovery (#2194)

Michael D'Angelofe1f1dc63eed
diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs
index ad94df7e8..70f5b8f65 100644
--- a/sdk/typescript/scripts/package-dist-files.mjs
+++ b/sdk/typescript/scripts/package-dist-files.mjs
@@ -76,6 +76,7 @@ const distModules = [
"finding-catalogue",
"findings-import",
"git-remote",
+ "local-git-metadata",
"github",
"github-advisories",
"github-dependabot-findings",
diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts
index 8be8c7bd5..7f648ae65 100644
--- a/sdk/typescript/src/external-findings-local.ts
+++ b/sdk/typescript/src/external-findings-local.ts
@@ -1,7 +1,4 @@
-import { lstat, readFile, realpath, stat } from "node:fs/promises";
-import { dirname, isAbsolute, join, resolve, sep } from "node:path";
import { isNumberObject } from "node:util/types";
-import { environmentEntry } from "./auth.js";
import { CodexSecurityError } from "./errors.js";
import {
forwardExternalImportSetting,
@@ -23,6 +20,10 @@ import type {
Provider,
} from "./external-import-models.js";
import { localFindingRepositories } from "./external-findings-read.js";
+import {
+ hasGitMetadata,
+ localGitProtectionRoots,
+} from "./local-git-metadata.js";
import {
blackDuckReportRepositories,
projectedBlackDuckRepositoryUrl,
@@ -53,11 +54,8 @@ import {
import {
abortable,
enclosingGitWorktreeRoot,
- gitMarkerRoot,
gitOutput,
- gitProtectionRoots,
normalizeRepository,
- nullIfMissingPath,
} from "./targets.js";
/** A scanner source record, independent of a Codex assessment or completed scan. */
@@ -405,194 +403,6 @@ function sameScannerRepositories(
return true;
}
-async function bareMetadataRoots(
- repository: string,
- signal?: AbortSignal,
-): Promise<string[]> {
- const roots: string[] = [];
- let directory = repository;
- for (;;) {
- signal?.throwIfAborted();
- const present = async (name: string) =>
- (await abortable(
- () => lstat(join(directory, name)).catch(nullIfMissingPath),
- signal,
- )) !== null;
- // Git also discovers bare repositories and linked metadata without .git.
- const head = await abortable(
- () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),
- signal,
- );
- if (
- (head?.isFile() || head?.isSymbolicLink()) &&
- ((await present("commondir")) ||
- ((await present("objects")) && (await present("refs"))))
- )
- roots.push(directory);
- const parent = dirname(directory);
- if (parent === directory) return roots;
- directory = parent;
- }
-}
-
-async function metadataPointer(
- path: string,
- prefix: string,
- context: { directory: string; environment: NodeJS.ProcessEnv },
- signal?: AbortSignal,
-): Promise<string | undefined> {
- const metadata = await abortable(
- () => stat(path).catch(nullIfMissingPath),
- signal,
- );
- if (!metadata?.isFile()) return undefined;
- const text = await abortable(() => readFile(path, "utf8"), signal);
- if (!text.startsWith(prefix) || (prefix === "" && text.length === 0))
- return undefined;
- const payload = text.slice(prefix.length).replace(/[\r\n]+$/u, "");
- if (prefix !== "" && payload.length === 0) return undefined;
- // Git checks emptiness before consuming the path as a NUL-terminated string.
- const value = payload.split("\0", 1)[0]!;
- // Preserve filesystem traversal through links before '..'.
- let target = isAbsolute(value) ? value : `${dirname(path)}${sep}${value}`;
- if (process.platform === "win32") {
- const drive = /^[A-Za-z]:(?![\\/])/u.exec(value)?.[0].toUpperCase();
- if (drive !== undefined) {
- const directory =
- context.directory.slice(0, 2).toUpperCase() === drive
- ? context.directory
- : environmentEntry(context.environment, `=${drive}`) || `${drive}\\`;
- target = resolve(directory, value.slice(2));
- } else if (isAbsolute(value)) target = resolve(context.directory, value);
- }
- return (
- (await abortable(
- () => realpath(target).catch(nullIfMissingPath),
- signal,
- )) ?? undefined
- );
-}
-
-async function associatedMetadataRoots(
- directory: string,
- context: { directory: string; environment: NodeJS.ProcessEnv },
- signal?: AbortSignal,
- configurationDirectory?: string,
-): Promise<string[]> {
- const head = await abortable(
- () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),
- signal,
- );
- if (!head || (!head.isFile() && !head.isSymbolicLink())) return [];
- const commonMarker = join(directory, "commondir");
- const hasCommon = await abortable(
- () => lstat(commonMarker).catch(nullIfMissingPath),
- signal,
- );
- const common = hasCommon
- ? await metadataPointer(commonMarker, "", context, signal)
- : undefined;
- if (hasCommon && common === undefined) return [];
- const storage = common ?? directory;
- for (const name of ["objects", "refs"]) {
- const metadata = await abortable(
- () => stat(join(storage, name)).catch(nullIfMissingPath),
- signal,
- );
- if (!metadata?.isDirectory()) return [];
- }
- const roots = common === undefined ? [directory] : [directory, common];
- if (
- process.platform === "win32" &&
- configurationDirectory !== undefined &&
- configurationDirectory !== context.directory
- ) {
- // Git rereads commondir after changing to the discovered repository root.
- roots.push(
- ...(await associatedMetadataRoots(
- directory,
- {
- ...context,
- directory: await abortable(
- () => canonicalConfigPath(configurationDirectory),
- signal,
- ),
- },
- signal,
- )),
- );
- }
- return roots;
-}
-
-export async function localGitProtectionRoots(
- repository: string,
- environment: NodeJS.ProcessEnv,
- signal?: AbortSignal,
-): Promise<string[]> {
- const roots = new Set(
- await abortable(() => gitProtectionRoots(repository, signal), signal),
- );
- const context = {
- directory: await abortable(() => canonicalConfigPath(repository), signal),
- environment,
- };
- const paths = new Set([resolve(repository), context.directory]);
- for (const path of paths) {
- for (const root of await bareMetadataRoots(path, signal)) {
- roots.add(root);
- for (const associated of await associatedMetadataRoots(
- root,
- context,
- signal,
- root,
- ))
- roots.add(associated);
- }
- let directory = path;
- for (;;) {
- signal?.throwIfAborted();
- const marker = join(directory, ".git");
- const metadata = await abortable(
- () => stat(marker).catch(nullIfMissingPath),
- signal,
- );
- if (metadata !== null) {
- roots.add(directory);
- const gitDirectory = metadata.isDirectory()
- ? await abortable(() => realpath(marker), signal)
- : await metadataPointer(marker, "gitdir: ", context, signal);
- if (gitDirectory !== undefined)
- for (const associated of await associatedMetadataRoots(
- gitDirectory,
- context,
- signal,
- directory,
- ))
- roots.add(associated);
- }
- const parent = dirname(directory);
- if (parent === directory) break;
- directory = parent;
- }
- }
- for (const root of [...roots])
- roots.add(await abortable(() => canonicalConfigPath(root), signal));
- return [...roots];
-}
-
-async function hasGitMetadata(
- repository: string,
- signal?: AbortSignal,
-): Promise<boolean> {
- return (
- (await abortable(
- () => gitMarkerRoot(repository, signal, "nearest"),
- signal,
- )) !== null || (await bareMetadataRoots(repository, signal)).length !== 0
- );
-}
-
/** @internal */
export interface ConfiguredFetchRemote {
name: string;
diff --git a/sdk/typescript/src/local-git-metadata.ts b/sdk/typescript/src/local-git-metadata.ts
new file mode 100644
index 000000000..bb80c834e
--- /dev/null
+++ b/sdk/typescript/src/local-git-metadata.ts
@@ -0,0 +1,198 @@
+import { lstat, readFile, realpath, stat } from "node:fs/promises";
+import { dirname, isAbsolute, join, resolve, sep } from "node:path";
+import { environmentEntry } from "./auth.js";
+import { canonicalConfigPath } from "./runtime.js";
+import {
+ abortable,
+ gitMarkerRoot,
+ gitProtectionRoots,
+ nullIfMissingPath,
+} from "./targets.js";
+
+async function bareMetadataRoots(
+ repository: string,
+ signal?: AbortSignal,
+): Promise<string[]> {
+ const roots: string[] = [];
+ let directory = repository;
+ for (;;) {
+ signal?.throwIfAborted();
+ const present = async (name: string) =>
+ (await abortable(
+ () => lstat(join(directory, name)).catch(nullIfMissingPath),
+ signal,
+ )) !== null;
+ // Git also discovers bare repositories and linked metadata without .git.
+ const head = await abortable(
+ () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),
+ signal,
+ );
+ if (
+ (head?.isFile() || head?.isSymbolicLink()) &&
+ ((await present("commondir")) ||
+ ((await present("objects")) && (await present("refs"))))
+ )
+ roots.push(directory);
+ const parent = dirname(directory);
+ if (parent === directory) return roots;
+ directory = parent;
+ }
+}
+
+async function metadataPointer(
+ path: string,
+ prefix: string,
+ context: { directory: string; environment: NodeJS.ProcessEnv },
+ signal?: AbortSignal,
+): Promise<string | undefined> {
+ const metadata = await abortable(
+ () => stat(path).catch(nullIfMissingPath),
+ signal,
+ );
+ if (!metadata?.isFile()) return undefined;
+ const text = await abortable(() => readFile(path, "utf8"), signal);
+ if (!text.startsWith(prefix) || (prefix === "" && text.length === 0))
+ return undefined;
+ const payload = text.slice(prefix.length).replace(/[\r\n]+$/u, "");
+ if (prefix !== "" && payload.length === 0) return undefined;
+ // Git checks emptiness before consuming the path as a NUL-terminated string.
+ const value = payload.split("\0", 1)[0]!;
+ // Preserve filesystem traversal through links before '..'.
+ let target = isAbsolute(value) ? value : `${dirname(path)}${sep}${value}`;
+ if (process.platform === "win32") {
+ const drive = /^[A-Za-z]:(?![\\/])/u.exec(value)?.[0].toUpperCase();
+ if (drive !== undefined) {
+ const directory =
+ context.directory.slice(0, 2).toUpperCase() === drive
+ ? context.directory
+ : environmentEntry(context.environment, `=${drive}`) || `${drive}\\`;
+ target = resolve(directory, value.slice(2));
+ } else if (isAbsolute(value)) target = resolve(context.directory, value);
+ }
+ return (
+ (await abortable(
+ () => realpath(target).catch(nullIfMissingPath),
+ signal,
+ )) ?? undefined
+ );
+}
+
+async function associatedMetadataRoots(
+ directory: string,
+ context: { directory: string; environment: NodeJS.ProcessEnv },
+ signal?: AbortSignal,
+ configurationDirectory?: string,
+): Promise<string[]> {
+ const head = await abortable(
+ () => lstat(join(directory, "HEAD")).catch(nullIfMissingPath),
+ signal,
+ );
+ if (!head || (!head.isFile() && !head.isSymbolicLink())) return [];
+ const commonMarker = join(directory, "commondir");
+ const hasCommon = await abortable(
+ () => lstat(commonMarker).catch(nullIfMissingPath),
+ signal,
+ );
+ const common = hasCommon
+ ? await metadataPointer(commonMarker, "", context, signal)
+ : undefined;
+ if (hasCommon && common === undefined) return [];
+ const storage = common ?? directory;
+ for (const name of ["objects", "refs"]) {
+ const metadata = await abortable(
+ () => stat(join(storage, name)).catch(nullIfMissingPath),
+ signal,
+ );
+ if (!metadata?.isDirectory()) return [];
+ }
+ const roots = common === undefined ? [directory] : [directory, common];
+ if (
+ process.platform === "win32" &&
+ configurationDirectory !== undefined &&
+ configurationDirectory !== context.directory
+ ) {
+ // Git rereads commondir after changing to the discovered repository root.
+ roots.push(
+ ...(await associatedMetadataRoots(
+ directory,
+ {
+ ...context,
+ directory: await abortable(
+ () => canonicalConfigPath(configurationDirectory),
+ signal,
+ ),
+ },
+ signal,
+ )),
+ );
+ }
+ return roots;
+}
+
+export async function localGitProtectionRoots(
+ repository: string,
+ environment: NodeJS.ProcessEnv,
+ signal?: AbortSignal,
+): Promise<string[]> {
+ const roots = new Set(
+ await abortable(() => gitProtectionRoots(repository, signal), signal),
+ );
+ const context = {
+ directory: await abortable(() => canonicalConfigPath(repository), signal),
+ environment,
+ };
+ const paths = new Set([resolve(repository), context.directory]);
+ for (const path of paths) {
+ for (const root of await bareMetadataRoots(path, signal)) {
+ roots.add(root);
+ for (const associated of await associatedMetadataRoots(
+ root,
+ context,
+ signal,
+ root,
+ ))
+ roots.add(associated);
+ }
+ let directory = path;
+ for (;;) {
+ signal?.throwIfAborted();
+ const marker = join(directory, ".git");
+ const metadata = await abortable(
+ () => stat(marker).catch(nullIfMissingPath),
+ signal,
+ );
+ if (metadata !== null) {
+ roots.add(directory);
+ const gitDirectory = metadata.isDirectory()
+ ? await abortable(() => realpath(marker), signal)
+ : await metadataPointer(marker, "gitdir: ", context, signal);
+ if (gitDirectory !== undefined)
+ for (const associated of await associatedMetadataRoots(
+ gitDirectory,
+ context,
+ signal,
+ directory,
+ ))
+ roots.add(associated);
+ }
+ const parent = dirname(directory);
+ if (parent === directory) break;
+ directory = parent;
+ }
+ }
+ for (const root of [...roots])
+ roots.add(await abortable(() => canonicalConfigPath(root), signal));
+ return [...roots];
+}
+
+export async function hasGitMetadata(
+ repository: string,
+ signal?: AbortSignal,
+): Promise<boolean> {
+ return (
+ (await abortable(
+ () => gitMarkerRoot(repository, signal, "nearest"),
+ signal,
+ )) !== null || (await bareMetadataRoots(repository, signal)).length !== 0
+ );
+}
diff --git a/sdk/typescript/src/sonar-collector.ts b/sdk/typescript/src/sonar-collector.ts
index ac66a98c5..e4377d401 100644
--- a/sdk/typescript/src/sonar-collector.ts
+++ b/sdk/typescript/src/sonar-collector.ts
@@ -6,8 +6,8 @@ import {
boundSonarWorktree,
sonarRepositoryForRemote,
sonarCheckoutMatchesSource,
- localGitProtectionRoots,
} from "./external-findings-local.js";
+import { localGitProtectionRoots } from "./local-git-metadata.js";
import { repositoryUrlKey } from "./external-import-contract.js";
import {
forwardExternalImportSetting,
diff --git a/sdk/typescript/src/sonar-import.ts b/sdk/typescript/src/sonar-import.ts
index 7daa8f728..a27005cbf 100644
--- a/sdk/typescript/src/sonar-import.ts
+++ b/sdk/typescript/src/sonar-import.ts
@@ -8,9 +8,9 @@ import { requireExternalImports } from "./external-imports-feature.js";
import {
configuredFetchRemotes,
type ConfiguredFetchRemote,
- localGitProtectionRoots,
prepareLocalImport,
} from "./external-findings-local.js";
+import { localGitProtectionRoots } from "./local-git-metadata.js";
import {
ExternalPublicationError,
externalPublicationStateDirectory,