Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: make verbose scan reruns and cost failures actionable (#253)

mldangelo-oai50e2ad027ddf
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index c10460560..2747672f4 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -62,6 +62,7 @@ import {
OutputInsideProtectedRootError,
PluginPythonUnavailableError,
redactedErrorMessage,
+ ScanCostLimitExceededError,
ScanInterruptedError,
} from "./errors.js";
import type { SeverityLevel } from "./models.js";
@@ -877,8 +878,14 @@ export async function main(
args: z.object({
scanId: z.string().min(1).describe("Saved scan identifier."),
}),
+ options: z.object({
+ verbose: z
+ .boolean()
+ .default(false)
+ .describe("Print scan diagnostics to stderr."),
+ }),
output: z.record(z.string(), z.unknown()).optional(),
- async run({ args, error: incurError }) {
+ async run({ args, error: incurError, options }) {
let scanArguments: ScanArguments;
try {
const { recipe } = await dependencies.runWorkbench([
@@ -887,6 +894,7 @@ export async function main(
args.scanId,
]);
scanArguments = scanArgumentsFromRecipe(recipe, args.scanId);
+ scanArguments.verbose = options.verbose;
} catch (error) {
const message = redactedErrorMessage(error);
errorOutput.write(`codex-security: ${message}\n`);
@@ -2732,6 +2740,7 @@ async function runScan(
codex_version: CODEX_EXECUTABLE_VERSION,
codex_sdk_version: CODEX_SDK_VERSION,
mode: arguments_.mode,
+ max_cost_usd: arguments_.maxCostUsd,
target:
arguments_.paths.length > 0
? "paths"
@@ -2950,15 +2959,22 @@ async function runScan(
};
}
if (failed) {
+ const costLimitFailure =
+ failure instanceof ScanCostLimitExceededError ? failure : undefined;
const message =
failure instanceof OutputInsideProtectedRootError
? redactedErrorMessage(protectedRootErrorMessage(failure))
: scanFailureMessage(failure, selectedAuthentication);
diagnostic("scan.failed", {
- classification: isLocalScanFailure(failure)
- ? "local"
- : classifyConnectionFailure(failure),
+ classification:
+ costLimitFailure !== undefined
+ ? "cost_limit_exceeded"
+ : isLocalScanFailure(failure)
+ ? "local"
+ : classifyConnectionFailure(failure),
partial_output: scanDir !== null,
+ max_cost_usd: costLimitFailure?.maxCostUsd,
+ estimated_usd: costLimitFailure?.cost.estimatedUsd,
});
errorOutput.write(`${message}\n`);
if (failure instanceof ScanInterruptedError) {
diff --git a/sdk/typescript/tests-ts/cli-authentication.test.ts b/sdk/typescript/tests-ts/cli-authentication.test.ts
index f2e4c00bd..287918eab 100644
--- a/sdk/typescript/tests-ts/cli-authentication.test.ts
+++ b/sdk/typescript/tests-ts/cli-authentication.test.ts
@@ -338,7 +338,15 @@ describe("CLI authentication", () => {
});
test("offers the existing interactive prompt when both sign-ins are available", async () => {
- for (const selection of ["chatgpt", "api-key"] as const) {
+ for (const [argv, selection] of [
+ [["scan"], "chatgpt"],
+ [["scan"], "api-key"],
+ [["scans", "rerun", "scan-original", "--verbose", "--json"], "chatgpt"],
+ [
+ ["scans", "rerun", "scan-original", "--verbose", "--format", "jsonl"],
+ "chatgpt",
+ ],
+ ] as const) {
const stderr = capture(true);
let selected: ScanOptions["auth"];
let question = "";
@@ -348,6 +356,14 @@ describe("CLI authentication", () => {
onTurn: (_repository, options) => {
selected = (options as ScanOptions).auth;
},
+ onWorkbench: () => ({
+ recipe: {
+ repository: "/original/repository",
+ target: { kind: "repository", paths: [] },
+ mode: "standard",
+ config: {},
+ },
+ }),
});
deps.hasStoredChatGPTSignIn = async () => true;
deps.scanAuthenticationPrompt = {
@@ -362,9 +378,7 @@ describe("CLI authentication", () => {
},
};
- expect(await main(["scan"], capture().stream, stderr.stream, deps)).toBe(
- 0,
- );
+ expect(await main(argv, capture().stream, stderr.stream, deps)).toBe(0);
expect(selected).toBe(selection);
expect(question).toBe("How would you like to authenticate this scan?");
expect(choices).toEqual([
diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts
index c63268899..b1eb88419 100644
--- a/sdk/typescript/tests-ts/cli.test.ts
+++ b/sdk/typescript/tests-ts/cli.test.ts
@@ -19,6 +19,7 @@ import { describe, expect, test } from "bun:test";
import type {
CodexSecurityConfig,
JsonObject,
+ ScanOptions,
ScanPreflight,
} from "../src/index.js";
import { redactedErrorMessage } from "../src/errors.js";
@@ -120,6 +121,20 @@ describe("CLI", () => {
},
});
+ const rerunSchema = capture();
+ expect(
+ await main(
+ ["scans", "rerun", "--schema", "--format", "json"],
+ rerunSchema.stream,
+ capture().stream,
+ dependencies(),
+ ),
+ ).toBe(0);
+ expect(JSON.parse(rerunSchema.text())).toMatchObject({
+ args: { properties: { scanId: { type: "string" } } },
+ options: { properties: { verbose: { type: "boolean" } } },
+ });
+
const matchSchema = capture();
expect(
await main(
@@ -2292,11 +2307,10 @@ describe("CLI", () => {
expect(
await main(
- ["scans", "rerun", "scan-original"],
+ ["scans", "rerun", "scan-original", "--verbose", "--json"],
stdout.stream,
stderr.stream,
dependencies({
- environment: { CODEX_SECURITY_LOG_LEVEL: "debug" },
onWorkbench: () => ({
recipe: {
repository: "/original/repository",
@@ -2311,9 +2325,12 @@ describe("CLI", () => {
}),
),
).toBe(0);
+ expect(JSON.parse(stdout.text())).toEqual(fakeResult().toJSON());
expect(stderr.text()).toContain(
"codex-security: debug: scan.configuration",
);
+ expect(stderr.text()).toContain("codex-security: debug: scan.started");
+ expect(stderr.text()).toContain("codex-security: debug: scan.completed");
expect(stderr.text()).toContain('model="gpt-original"');
expect(stderr.text()).toContain('reasoning_effort="high"');
});
@@ -3725,7 +3742,7 @@ describe("CLI", () => {
expect(stderr.text()).toContain("cache_write_input_tokens=200");
});
- test("reports a scan stopped when its live cost exceeds the limit", async () => {
+ test("reports and classifies a scan stopped when its live cost exceeds the limit", async () => {
const stdout = capture();
const stderr = capture();
const cost = fakeResult([], "complete", {
@@ -3736,11 +3753,12 @@ describe("CLI", () => {
expect(
await main(
- ["scan", ".", "--json", "--max-cost", "0.005"],
+ ["scan", ".", "--verbose", "--json", "--max-cost", "0.005"],
stdout.stream,
stderr.stream,
dependencies({
- onTurn: () => {
+ onTurn: (_repository, options) => {
+ (options as ScanOptions).onOutputDirReady?.("/tmp/scan");
throw new ScanCostLimitExceededError(0.005, cost, "/tmp/scan");
},
}),
@@ -3750,6 +3768,12 @@ describe("CLI", () => {
expect(stderr.text()).toContain(
"Scan stopped: estimated cost $0.00625 exceeded the $0.005 limit; partial output remains at /tmp/scan.",
);
+ expect(stderr.text()).toMatch(
+ /scan\.configuration[^\n]*max_cost_usd=0\.005/u,
+ );
+ expect(stderr.text()).toContain(
+ 'scan.failed classification="cost_limit_exceeded" partial_output=true max_cost_usd=0.005 estimated_usd=0.00625',
+ );
});
test("accepts a scan at its estimated cost limit", async () => {