Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(cli): show active reviews and result consolidation during Deep Scan (#1253)

mldangelo-oai1163374f6c2a
diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py
index 0c3c7ce6b..d1d0b2dca 100644
--- a/plugins/codex-security/scripts/deep_scan_workbench.py
+++ b/plugins/codex-security/scripts/deep_scan_workbench.py
@@ -437,7 +437,7 @@ def independent_review_progress(
"active": int(active),
"completed": int(run["completion_sequence"]),
"maximum": int(run["max_discovery_runs"]),
- "consolidating": run["phase"] == "reducing",
+ "consolidating": run["phase"] in {"reducing", "terminal"},
"updatedAt": str(run["updated_at"]),
}
diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py
index a0ff84bf4..491655d8c 100644
--- a/plugins/codex-security/tests/test_workbench_deep_scan.py
+++ b/plugins/codex-security/tests/test_workbench_deep_scan.py
@@ -1971,6 +1971,12 @@ def test_target_continuation_reuses_terminal_coordinator_across_threads(
assert continued["startDisposition"] == "joined"
assert continued["deepScan"]["scanId"] == scan_id
assert continued["deepScan"]["manifestPath"] == str(manifest)
+ assert get_scan(state_dir, scan_id)["scan"]["progress"]["independentReviews"] == {
+ "active": 0,
+ "completed": 0,
+ "maximum": 40,
+ "consolidating": True,
+ }
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
assert connection.execute("SELECT COUNT(*) FROM workspaces").fetchone() == (1,)
assert connection.execute("SELECT COUNT(*) FROM scans").fetchone() == (1,)
@@ -2980,6 +2986,12 @@ def test_failure_capped_completion_preserves_partial_results_and_exact_omissions
assert finished["terminalReason"] == "capped"
assert finished["config"]["maxDiscoveryRuns"] == 10
assert finished["dispatchedCount"] == 4
+ assert get_scan(state_dir, scan_id)["scan"]["progress"]["independentReviews"] == {
+ "active": 0,
+ "completed": 3,
+ "maximum": 10,
+ "consolidating": True,
+ }
assert json.loads((scan_dir / "coverage.json").read_text())["completeness"] == "partial"
assert len(json.loads((scan_dir / "findings.json").read_text())["findings"]) == 1
assert (
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index 79d88bf3e..c14e25f77 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -311,7 +311,9 @@ dispatch status does not mean delegation was skipped. `onSessionEvent` receives
saved events with thread IDs and worker numbers and can contain source code or
credentials. Deep scans additionally expose durable independent-review counts
through `onDeepProgress`: `completed`, `active`, and `maximum`. The maximum is a
-configured cap, not a percentage denominator. `ScanOptions` lists all callbacks.
+configured cap, not a percentage denominator. The optional `consolidating` flag
+reports when results are being combined or the coordinator has finished.
+`ScanOptions` lists all callbacks.
Costs estimate API-equivalent model usage, not your bill or ChatGPT subscription
allowance. Use `cost.estimatedUsdRange` for reporting. `maxCostUsd` uses the
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index e27d36891..de2772ed2 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -87,6 +87,7 @@ import {
readCodexHomeConfig,
} from "./auth.js";
import { loadContract, sha256Text } from "./contract.js";
+import type { DeepScanProgress } from "./deep-progress.js";
import { isRecord as isJsonObject } from "./record.js";
import { suggestOwnersInternal } from "./suggest-owners.js";
import { parseImportedFindings } from "./findings-import.js";
@@ -8092,6 +8093,8 @@ async function executeScan(
let lastProgressUpdate = "";
let workerCapacity: { planned: number; started: number } | null = null;
let fileProgress: ScanProgress | null = null;
+ let deepProgress: DeepScanProgress | null = null;
+ let deepConsolidating = false;
let runningCost: Readonly<ScanCost> | null = null;
let maxCostUsd = arguments_.maxCostUsd;
const showCost = arguments_.showCost === true || maxCostUsd !== undefined;
@@ -8309,7 +8312,11 @@ async function executeScan(
`Workers: ${workerCapacity.started}/${workerCapacity.planned}`,
);
}
- if (fileProgress !== null && fileProgress.filesTotal > 0) {
+ if (deepProgress !== null) {
+ details.push(
+ `Reviews: ${deepProgress.completed} completed, ${deepProgress.active} active, cap ${deepProgress.maximum}`,
+ );
+ } else if (fileProgress !== null && fileProgress.filesTotal > 0) {
details.push(
`Files: ${fileProgress.filesCompleted.toLocaleString("en-US")}/${fileProgress.filesTotal.toLocaleString("en-US")}`,
);
@@ -8539,6 +8546,30 @@ async function executeScan(
);
progress.startTimer(runningMessage());
},
+ onDeepProgress: (update) => {
+ if (
+ deepProgress === null &&
+ update.completed === 0 &&
+ update.active === 0 &&
+ !update.consolidating
+ )
+ return;
+ deepProgress = update;
+ deepConsolidating =
+ update.consolidating === true ||
+ (deepConsolidating && update.active === 0);
+ phase = deepConsolidating ? "consolidating results" : "discovery";
+ const message = `Scan phase: ${phase} | Reviews: ${update.completed} completed, ${update.active} active, cap ${update.maximum}`;
+ if (dashboard !== null) {
+ dashboard.setStage(phase);
+ dashboard.note(message);
+ return;
+ }
+ if (progress === null) return;
+ progress.stopTimer();
+ progress.stage(message);
+ progress.startTimer(runningMessage());
+ },
onWorkerStatus: (status) => {
const update =
status.kind === "preflight"
diff --git a/sdk/typescript/src/deep-progress.ts b/sdk/typescript/src/deep-progress.ts
index 574de2740..9800dfcb6 100644
--- a/sdk/typescript/src/deep-progress.ts
+++ b/sdk/typescript/src/deep-progress.ts
@@ -5,6 +5,8 @@ export interface DeepScanProgress {
completed: number;
active: number;
maximum: number;
+ /** Whether the coordinator is reducing results or has finished. */
+ consolidating?: boolean;
}
interface DeepScanProgressTrackerOptions {
@@ -85,14 +87,20 @@ export function deepScanProgressFromWorkbench(
const independentReviews = progress["independentReviews"];
if (independentReviews === undefined) return null;
if (isRecord(independentReviews)) {
- const { completed, active, maximum } = independentReviews;
+ const { completed, active, maximum, consolidating } = independentReviews;
if (
isSafeNonNegativeInteger(completed) &&
isSafeNonNegativeInteger(active) &&
isSafeNonNegativeInteger(maximum) &&
- maximum > 0
+ maximum > 0 &&
+ (consolidating === undefined || typeof consolidating === "boolean")
) {
- return { completed, active, maximum };
+ return {
+ completed,
+ active,
+ maximum,
+ ...(typeof consolidating === "boolean" ? { consolidating } : {}),
+ };
}
}
throw new Error(
@@ -108,6 +116,7 @@ function sameProgress(
right !== null &&
left.completed === right.completed &&
left.active === right.active &&
- left.maximum === right.maximum
+ left.maximum === right.maximum &&
+ left.consolidating === right.consolidating
);
}
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index d50e04c59..d3d061d77 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -3041,53 +3041,79 @@ describe("CodexSecurity orchestration", () => {
await client.close();
});
- test("forwards durable Deep Scan independent-review progress", async () => {
- const { repository, codexHome, scanDir } = await scanDirectories();
- const updates: DeepScanProgress[] = [];
- const environment = { CODEX_CLI_PATH: process.execPath };
- const client = TestClient.withDependencies({
- environment,
- prepareRuntime: runtimePreparer(codexHome, () => ({ environment })),
- resolvePluginPython: async () => "/managed/python",
- prepareOutputDir: async () => scanDir,
- repositoryRevision: async () => "deadbeef",
- resolveCodexCommand: () => ({ command: process.execPath }),
- runWorkbench: async (
- _options: unknown,
- args: readonly string[],
- input?: string,
- ): Promise<JsonObject> => {
- if (args[0] === "get-scan") {
- return {
- scan: {
- progress: {
- independentReviews: {
- completed: 3,
- active: 2,
- maximum: 40,
+ test.each([true, "false", null])(
+ "routes durable Deep progress without replacing scan errors (consolidating=%p)",
+ async (consolidating) => {
+ const { repository, codexHome, scanDir } = await scanDirectories();
+ const updates: DeepScanProgress[] = [];
+ const observerErrors: unknown[] = [];
+ const warnings: string[] = [];
+ const environment = { CODEX_CLI_PATH: process.execPath };
+ const client = TestClient.withDependencies({
+ environment,
+ prepareRuntime: runtimePreparer(codexHome, () => ({ environment })),
+ resolvePluginPython: async () => "/managed/python",
+ prepareOutputDir: async () => scanDir,
+ repositoryRevision: async () => "deadbeef",
+ resolveCodexCommand: () => ({ command: process.execPath }),
+ runWorkbench: async (
+ _options: unknown,
+ args: readonly string[],
+ input?: string,
+ ): Promise<JsonObject> => {
+ if (args[0] === "get-scan") {
+ return {
+ scan: {
+ progress: {
+ independentReviews: {
+ completed: 3,
+ active: 2,
+ maximum: 40,
+ consolidating,
+ },
},
},
- },
- };
- }
- return mockWorkbench(args, input);
- },
- createCodex: codexFactory(async () => {
- await Bun.sleep(0);
- throw new Error("deep progress captured");
- }),
- });
+ };
+ }
+ return mockWorkbench(args, input);
+ },
+ createCodex: codexFactory(async () => {
+ await Bun.sleep(0);
+ throw new Error("deep progress captured");
+ }),
+ });
- await expect(
- client.run(repository, {
- mode: "deep",
- onDeepProgress: (progress) => updates.push(progress),
- }),
- ).rejects.toThrow("deep progress captured");
- await Bun.sleep(0);
- expect(updates).toEqual([{ completed: 3, active: 2, maximum: 40 }]);
- await client.close();
- });
+ await expect(
+ client.run(repository, {
+ mode: "deep",
+ onDeepProgress: (progress) => {
+ updates.push(progress);
+ throw new Error("progress presenter failed");
+ },
+ onWarning: (warning) => warnings.push(warning),
+ onObserverError: (observer, error) =>
+ observerErrors.push([observer, error]),
+ }),
+ ).rejects.toThrow("deep progress captured");
+ await Bun.sleep(0);
+ if (consolidating === true) {
+ expect(updates).toEqual([
+ { completed: 3, active: 2, maximum: 40, consolidating: true },
+ ]);
+ expect(observerErrors).toEqual([
+ ["onDeepProgress", new Error("progress presenter failed")],
+ ]);
+ expect(warnings).toEqual([]);
+ } else {
+ expect(updates).toEqual([]);
+ expect(observerErrors).toEqual([]);
+ expect(warnings).toEqual([
+ "Could not track Deep Scan progress: Codex Security workbench returned invalid Deep Scan progress.",
+ ]);
+ }
+ await client.close();
+ },
+ );
test.skipIf(process.platform !== "win32")(
"loads deep scan settings from a backslash home-relative CODEX_HOME",
diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts
index dd5eb6d75..47363b5d3 100644
--- a/sdk/typescript/tests-ts/cli.test.ts
+++ b/sdk/typescript/tests-ts/cli.test.ts
@@ -2042,6 +2042,111 @@ describe("CLI", () => {
expect(text).not.toContain("Estimated cost: $0.0248865 of $2.00 limit");
});
+ test.each([false, true])(
+ "shows durable Deep progress without changing stdout or TUI layout (interactive=%s)",
+ async (interactive) => {
+ const { stdout, stderr, runCli } = createCliTest(main, {
+ stderr: interactive,
+ });
+ const deps = dependencies({ environment: { NO_COLOR: "1" } });
+ deps.createSecurity = () =>
+ fakeSecurity(async (_repository, options) => {
+ options?.onProgress?.({
+ phase: "preflight",
+ filesCompleted: 0,
+ filesTotal: 128,
+ });
+ expect(options?.onDeepProgress).toBeFunction();
+ const initial = stderr.text();
+ options?.onDeepProgress?.({
+ completed: 0,
+ active: 0,
+ maximum: 16,
+ consolidating: false,
+ });
+ expect(stderr.text()).toBe(initial);
+ options?.onDeepProgress?.({
+ completed: 0,
+ active: 4,
+ maximum: 16,
+ consolidating: false,
+ });
+ expect(stripVTControlCharacters(stderr.text())).toContain(
+ "discovery",
+ );
+ options?.onDeepProgress?.({
+ completed: 2,
+ active: 2,
+ maximum: 16,
+ consolidating: true,
+ });
+ expect(stripVTControlCharacters(stderr.text())).toContain(
+ "consolidating results",
+ );
+ const beforeTerminal = stderr.text().length;
+ options?.onDeepProgress?.({
+ completed: 4,
+ active: 0,
+ maximum: 16,
+ consolidating: false,
+ });
+ const terminal = stripVTControlCharacters(
+ stderr.text().slice(beforeTerminal),
+ );
+ expect(terminal).toContain("consolidating results");
+ expect(terminal).not.toContain("discovery");
+ const beforeDiscovery = stderr.text().length;
+ options?.onDeepProgress?.({
+ completed: 4,
+ active: 2,
+ maximum: 16,
+ consolidating: false,
+ });
+ const resumedDiscovery = stripVTControlCharacters(
+ stderr.text().slice(beforeDiscovery),
+ );
+ expect(resumedDiscovery).toContain("discovery");
+ expect(resumedDiscovery).not.toContain("consolidating results");
+ options?.onDeepProgress?.({
+ completed: 6,
+ active: 0,
+ maximum: 16,
+ consolidating: true,
+ });
+ return fakeResult();
+ });
+
+ expect(
+ await runCli(
+ [
+ "scan",
+ ".",
+ "--mode",
+ "deep",
+ ...(interactive ? [] : ["--headless"]),
+ "--json",
+ ],
+ deps,
+ ),
+ ).toBe(0);
+ expect(JSON.parse(stdout.text())).toEqual(fakeResult().toJSON());
+ const text = stripVTControlCharacters(stderr.text());
+ expect(text).toContain("Reviews: 0 completed, 4 active, cap 16");
+ expect(text).toContain("Reviews: 2 completed, 2 active, cap 16");
+ if (interactive) {
+ expect(text).not.toContain("STAGE");
+ expect(text).not.toContain("FILES");
+ } else {
+ const running = text
+ .split("\n")
+ .filter((line) => line.includes("Running scan:"))
+ .at(-1);
+ expect(running).toContain("consolidating results");
+ expect(running).not.toContain("Files:");
+ }
+ },
+ );
+
test("omits stage and file counts from interactive Deep scan dashboards", async () => {
const { stderr, runCli } = createCliTest(main, { stderr: true });
diff --git a/sdk/typescript/tests-ts/deep-progress.test.ts b/sdk/typescript/tests-ts/deep-progress.test.ts
index 7bd8a6269..e0a5a7026 100644
--- a/sdk/typescript/tests-ts/deep-progress.test.ts
+++ b/sdk/typescript/tests-ts/deep-progress.test.ts
@@ -29,7 +29,7 @@ describe("Deep Scan progress", () => {
consolidating: false,
}),
),
- ).toEqual({ completed: 34, active: 2, maximum: 40 });
+ ).toEqual({ completed: 34, active: 2, maximum: 40, consolidating: false });
});
test("waits for the Deep Scan coordinator to create independent-review state", () => {
@@ -40,6 +40,8 @@ describe("Deep Scan progress", () => {
{ completed: -1, active: 0, maximum: 40 },
{ completed: 0, active: 0, maximum: 0 },
{ completed: 0, active: "two", maximum: 40 },
+ { completed: 2, active: 0, maximum: 40, consolidating: "false" },
+ { completed: 2, active: 0, maximum: 40, consolidating: null },
])("rejects invalid workbench progress %#", (independentReviews) => {
expect(() =>
deepScanProgressFromWorkbench(workbenchResult(independentReviews)),
@@ -99,4 +101,22 @@ describe("Deep Scan progress", () => {
expect(aborted).toBe(true);
expect(progress).toEqual([]);
});
+
+ test("reports reduction transitions when review counts remain unchanged", async () => {
+ const reads = [false, true, true, false].map((consolidating) =>
+ workbenchResult({ completed: 2, active: 2, maximum: 40, consolidating }),
+ );
+ const progress: DeepScanProgress[] = [];
+ const tracker = new DeepScanProgressTracker({
+ read: async () => reads.shift(),
+ onProgress: (update) => progress.push(update),
+ });
+ for (let index = 0; index < 4; index++) await tracker.refresh();
+ tracker.stop();
+ expect(progress.map((update) => update.consolidating)).toEqual([
+ false,
+ true,
+ false,
+ ]);
+ });
});
diff --git a/sdk/typescript/tests-ts/scan-resume.test.ts b/sdk/typescript/tests-ts/scan-resume.test.ts
index ef7ca3deb..0661d058f 100644
--- a/sdk/typescript/tests-ts/scan-resume.test.ts
+++ b/sdk/typescript/tests-ts/scan-resume.test.ts
@@ -2,6 +2,7 @@ import { createCliTest } from "./support/cli-run.js";
import { gitText } from "./support/shell.js";
import { readJsonLines } from "./support/json.js";
import { randomUUID } from "node:crypto";
+import { stripVTControlCharacters } from "node:util";
import {
appendFile,
cp,
@@ -520,6 +521,54 @@ async function finishDiscovery(f: Awaited<ReturnType<typeof interruptedScan>>) {
]);
}
+test.each([false, true])(
+ "resumed CLI starts with terminal Deep progress (interactive=%p)",
+ async (interactive) => {
+ const f = await interruptedScan();
+ await finishDiscovery(f);
+ const { stderr, runCli } = createCliTest(main, { stderr: interactive });
+ const progress = Promise.withResolvers<void>();
+ const deps = resumeDependencies(f, () => ({
+ startThread: () => fail("Unexpected new session"),
+ resumeThread(threadId) {
+ expect(threadId).toBe(f.threadId);
+ return {
+ id: threadId,
+ async runStreamed() {
+ await progress.promise;
+ const text = stripVTControlCharacters(stderr.text()).replace(
+ /\s+/gu,
+ " ",
+ );
+ expect(text).toContain("consolidating results");
+ expect(text).toContain("Reviews: 0 completed, 0 active, cap 40");
+ expect(text).not.toContain("Scan phase: discovery");
+ throw new Error("Terminal progress captured");
+ },
+ };
+ },
+ }));
+ const code = await runCli(["scans", "resume", f.scanId, "--json"], {
+ ...deps,
+ createSecurity: (config) => {
+ const security = deps.createSecurity(config);
+ const run = security.run.bind(security);
+ security.run = (repository, options = {}) =>
+ run(repository, {
+ ...options,
+ onDeepProgress(update) {
+ options.onDeepProgress?.(update);
+ progress.resolve();
+ },
+ });
+ return security;
+ },
+ });
+ expect(code).toBe(2);
+ expect(stderr.text()).toContain("Terminal progress captured");
+ },
+);
+
test.each([
[false, false],
[true, false],