Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

feat(action): report usage and extend integration attribution (#1703)

Michael D'Angelo38a31ff6481f
diff --git a/examples/azure-pipelines/azure-pipelines.yml b/examples/azure-pipelines/azure-pipelines.yml
index 71d085dad..bb6e85b6f 100644
--- a/examples/azure-pipelines/azure-pipelines.yml
+++ b/examples/azure-pipelines/azure-pipelines.yml
@@ -93,6 +93,7 @@ jobs:
mkdir -p "$REPORT_DIRECTORY"
args=(scan "$TARGET_DIRECTORY" --provider amazon-bedrock
--model "$BEDROCK_MODEL_ID" --mode standard --effort high
+ --codex 'responses_api_metadata.codex_security_integration="azure_pipelines"'
--output-dir "$SCAN_DIRECTORY" --json)
if [[ "$SCAN_MODE" == diff ]]; then
args+=(--diff "$BASE_REVISION")
diff --git a/examples/github-actions/codex-security.yml b/examples/github-actions/codex-security.yml
index 23accb76e..3f7ee9feb 100644
--- a/examples/github-actions/codex-security.yml
+++ b/examples/github-actions/codex-security.yml
@@ -94,6 +94,7 @@ jobs:
--model "$BEDROCK_MODEL_ID"
--mode standard
--effort high
+ --codex 'responses_api_metadata.codex_security_integration="github_actions_workflow"'
--output-dir "$RUNNER_TEMP/codex-security-scan"
--json
)
diff --git a/github-action/README.md b/github-action/README.md
index 7373083ec..7ab23d00e 100644
--- a/github-action/README.md
+++ b/github-action/README.md
@@ -348,6 +348,11 @@ The Action writes a job summary and source annotations. Set
`upload-artifacts: 'true'` for downloadable reports, retained for seven days by
default. Reports can contain source code and vulnerability details.
+The job summary includes the configured provider, reported input/cached/output
+token usage, and estimated USD cost, including a range when supplied by the CLI.
+Missing measurements or pricing appear as `unavailable`; measured zeros remain
+zero. Cost estimates are not billing totals.
+
The Action uses the normal Actions job check. Its summary and final log message
distinguish findings above the failure threshold from a scan that could not
complete or a required reporting failure. With `fail-on-severity: none`, findings
@@ -469,7 +474,7 @@ npm --prefix github-action ci --ignore-scripts --no-audit --no-fund
npm --prefix github-action run docs
npm --prefix github-action run build
npm --prefix github-action run validate
-# Test the published CLI with synthetic scans; no model calls:
+# Test the published CLI with synthetic scans and a local request fixture:
npm --prefix github-action/runtime ci --ignore-scripts --no-audit --no-fund
npm --prefix github-action run test:cli
# Linux x64 with Node 24; no model calls:
@@ -479,8 +484,10 @@ node github-action/scripts/linux-smoke.mjs
Commit source changes and the generated `dist/*.cjs` bundles together.
Validation checks types, tests, Action metadata, documentation, and bundle
reproducibility. CI also tests the result adapter against the pinned CLI and its
-SARIF exporter using synthetic scans, runs the packaged Linux smoke test, and
-audits the Action and CLI dependency locks. The `@openai/codex-security` dependency in
+SARIF exporter using synthetic scans. A local inference fixture checks request
+attribution and the analytics opt-out with synthetic credentials; it makes no
+external model calls. CI also runs the packaged Linux smoke test and audits the
+Action and CLI dependency locks. The `@openai/codex-security` dependency in
`runtime/package.json` is the installed CLI version source. Main keeps an already
published CLI baseline so its checks remain runnable while a release PR prepares
the next product version. This development pin does not necessarily track the
diff --git a/github-action/dist/index.cjs b/github-action/dist/index.cjs
index 70cd64d7b..8a9b03350 100644
--- a/github-action/dist/index.cjs
+++ b/github-action/dist/index.cjs
@@ -56232,7 +56232,10 @@ async function analyzeResults(options) {
}
result.counts = { critical: 0, high: 0, medium: 0, low: 0, informational: 0 };
for (const finding of result.findings) result.counts[finding.severity] += 1;
+ const usage = value.turn?.usage;
+ result.usage = usage == null ? void 0 : { ...usage, cached_input_tokens: usage.cached_input_tokens ?? 0 };
result.estimatedCost = value.cost?.estimatedUsd;
+ result.estimatedCostRange = value.cost?.estimatedUsdRange;
result.paths = {
resultsDirectory: options.resultsDirectory,
manifestPath: (0, import_node_path5.join)(options.resultsDirectory, "scan-manifest.json"),
@@ -99228,12 +99231,17 @@ function resultTitle(result, inputs) {
function resultSummary(result, inputs, target) {
const esc = (value, limit = 4e3) => html(plain(value, limit));
const counts = result.counts ? Object.entries(result.counts).map(([level, count]) => `${level}: ${count}`).join(" \xB7 ") : "unavailable";
+ const usage = result.usage;
+ const tokens = usage ? `${usage.input_tokens.toLocaleString("en-US")} input (${usage.cached_input_tokens.toLocaleString("en-US")} cached) \xB7 ${usage.output_tokens.toLocaleString("en-US")} output` : "unavailable";
+ const range2 = result.estimatedCostRange;
+ const cost = range2 && range2.max !== range2.min ? range2.max === null ? `$${range2.min.toFixed(4)} (upper estimate unavailable)` : `$${range2.min.toFixed(4)}\u2013$${range2.max.toFixed(4)}` : result.estimatedCost === void 0 ? "unavailable" : `$${result.estimatedCost.toFixed(4)}`;
const parts = [
`**Scan:** ${result.scanStatus} \xB7 **Findings policy:** ${result.policyStatus} \xB7 **Report:** ${result.reportStatus}`,
`**Findings:** ${counts}`,
`**Scope:** ${inputs.scope}${inputs.paths.length ? ` <code>${esc(inputs.paths.join(", "))}</code>` : ""} \xB7 **Mode:** ${inputs.mode}`,
`**Commit:** <code>${target.scannedSha}</code>`,
- `**Model:** <code>${esc(inputs.model)}</code> \xB7 **Reasoning effort:** ${inputs.effort}`,
+ `**Configured provider:** <code>${esc(inputs.provider)}</code> \xB7 **Model:** <code>${esc(inputs.model)}</code> \xB7 **Reasoning effort:** ${inputs.effort}`,
+ `**Tokens:** ${tokens} \xB7 **Estimated cost:** ${cost}`,
`**Failure threshold:** ${inputs.failOnSeverity === "none" ? "report-only findings" : inputs.failOnSeverity + " and above"}. Findings at or above the configured threshold fail the action. Scanner and required reporting errors also fail the action.`,
...inputs.maxCost !== void 0 ? [`**Stop threshold:** $${inputs.maxCost} (estimated; in-flight requests can exceed it)`] : [],
"Applicable root and nested SECURITY.md policy is discovered by the scanner."
diff --git a/github-action/scripts/cli-integration.ts b/github-action/scripts/cli-integration.ts
index 7a76ee3ad..e62524c8e 100644
--- a/github-action/scripts/cli-integration.ts
+++ b/github-action/scripts/cli-integration.ts
@@ -1,9 +1,11 @@
-// Exercise the locked CLI or an installed release candidate without model calls.
+// Exercise the locked CLI or an installed release candidate with synthetic local requests only.
import assert from 'node:assert/strict';
-import { execFileSync, spawnSync } from 'node:child_process';
+import { execFile, execFileSync, spawnSync } from 'node:child_process';
import { copyFile, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises';
+import { createServer, type IncomingHttpHeaders } from 'node:http';
import { tmpdir } from 'node:os';
import { delimiter, dirname, join, resolve } from 'node:path';
+import { promisify } from 'node:util';
import { parseInputs, scanArguments } from '../src/inputs.js';
import { analyzeResults } from '../src/results.js';
import { captureRunnerPath, checkPython, resolveTool, runtimeEnvironment, writeRuntimeLauncher } from '../src/runtime.js';
@@ -75,6 +77,8 @@ try {
assert.deepEqual(result.findings.map(finding => finding.title),
cliResult.findings.findings.map((finding: any) => finding.title));
assert.equal(result.estimatedCost, cliResult.cost?.estimatedUsd);
+ assert.deepEqual(result.usage, cliResult.turn?.usage ?? undefined);
+ assert.deepEqual(result.estimatedCostRange, cliResult.cost?.estimatedUsdRange);
}
// Deep Scan cannot use --mock; validate the Action's actual arguments without model calls.
@@ -114,6 +118,67 @@ try {
assert.match(missing.stderr + missing.stdout, /--model is required when using --provider/);
}
+ // Exercise Action arguments through the installed CLI and native request serializer.
+ // A terminal fixture error stops each scan before any model output or tool execution.
+ const requests: Array<{headers: IncomingHttpHeaders; body: any}> = [];
+ const server = createServer(async (request, response) => {
+ const chunks: Buffer[] = [];
+ for await (const chunk of request) chunks.push(chunk);
+ if (request.method === 'POST' && request.url?.endsWith('/responses')) {
+ requests.push({headers:request.headers, body:JSON.parse(Buffer.concat(chunks).toString())});
+ response.writeHead(400, {'content-type':'application/json'});
+ response.end(JSON.stringify({error:{message:'Synthetic attribution fixture', type:'invalid_request_error'}}));
+ } else {
+ // Tell native WebSocket negotiation to fall back to HTTP.
+ response.writeHead(request.url?.endsWith('/responses') ? 426 : 200, {'content-type':'application/json'});
+ response.end('{"models":[]}');
+ }
+ });
+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+ try {
+ const address = server.address();
+ assert.ok(address && typeof address !== 'string');
+ const endpoint = `http://127.0.0.1:${address.port}`;
+ for (const provider of ['openai', 'amazon-bedrock']) {
+ requests.length = 0;
+ const requestHome = join(root, `request-${provider}`);
+ const codexHome = join(requestHome, '.codex');
+ await mkdir(codexHome, {recursive:true, mode:0o700});
+ const values: Record<string, string> = {provider, model:'gpt-5.6-sol', verbose:'false'};
+ const args = scanArguments(parseInputs(name => values[name] ?? '', repository),
+ {repository}, join(root, `request-${provider}-results`));
+ for (const override of [
+ provider === 'openai' ? `openai_base_url="${endpoint}/v1"`
+ : `model_providers.amazon-bedrock.base_url="${endpoint}/v1"`,
+ `chatgpt_base_url="${endpoint}"`, 'features.enable_request_compression=false',
+ 'otel.exporter="none"', 'otel.metrics_exporter="none"', 'otel.trace_exporter="none"',
+ ]) args.push('--codex', override);
+ const failure = await promisify(execFile)(process.execPath, [cli, ...args], {
+ cwd:repository, timeout:120_000, maxBuffer:8 * 1024 * 1024,
+ env:{...env, HOME:requestHome, CODEX_HOME:codexHome,
+ CODEX_SECURITY_STATE_DIR:join(requestHome, 'state'),
+ HTTP_PROXY:endpoint, HTTPS_PROXY:endpoint, ALL_PROXY:endpoint, NO_PROXY:'127.0.0.1',
+ OPENAI_API_KEY:'synthetic-api-key',
+ AWS_BEARER_TOKEN_BEDROCK:'synthetic-bedrock-token', AWS_REGION:'us-east-1'},
+ }).then(() => assert.fail('The synthetic request must fail'), error => error);
+ assert.equal(failure.code, 2, failure.stderr);
+ assert.equal(requests.length, 1, failure.stderr);
+ const request = requests[0];
+ for (const metadata of [
+ JSON.parse(String(request.headers['x-codex-turn-metadata'])),
+ JSON.parse(request.body.client_metadata['x-codex-turn-metadata']),
+ ]) {
+ assert.equal(metadata.codex_security_integration, 'github_action');
+ assert.equal(metadata.codex_security_provider, provider);
+ assert.equal(metadata.analytics_enabled, false);
+ assert.equal(metadata.thread_source, 'security_scan');
+ }
+ }
+ } finally {
+ server.closeAllConnections();
+ await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
+ }
+
// Normalization must not let a literal repository path become a framework option.
await mkdir(join(repository, '--help'));
await writeFile(join(repository, '--help/example.ts'), 'export const example = 1;\n');
@@ -253,7 +318,7 @@ try {
assert.equal(result.policyStatus, 'not-evaluated');
assert.equal(result.sarifUploadReady, false);
assert.ok(result.errors.some(error => error.includes(cliError.message)));
- console.log(`${candidatePackage === undefined ? 'Pinned' : 'Candidate'} CLI ${installed.version}: real JSON results, severity exits, SARIF export, Deep Scan, literal-path and virtualenv preflights, sandbox readiness and filesystem enforcement, MCP runner tracking, and failures passed without model calls.`);
+ console.log(`${candidatePackage === undefined ? 'Pinned' : 'Candidate'} CLI ${installed.version}: real JSON results, severity exits, SARIF export, Deep Scan, literal-path and virtualenv preflights, sandbox readiness and filesystem enforcement, MCP runner tracking, request attribution and analytics opt-outs, and failures passed without external model calls.`);
} finally {
await rm(root, { recursive: true, force: true });
}
diff --git a/github-action/src/reporting.ts b/github-action/src/reporting.ts
index 4c79d74ab..c3357c8e9 100644
--- a/github-action/src/reporting.ts
+++ b/github-action/src/reporting.ts
@@ -22,12 +22,19 @@ export function resultTitle(result: ScanResults, inputs: Inputs): string {
export function resultSummary(result: ScanResults, inputs: Inputs, target: Target): string {
const esc = (value: string, limit = 4000) => html(plain(value, limit));
const counts = result.counts ? Object.entries(result.counts).map(([level, count]) => `${level}: ${count}`).join(' · ') : 'unavailable';
+ const usage = result.usage;
+ const tokens = usage ? `${usage.input_tokens.toLocaleString('en-US')} input (${usage.cached_input_tokens.toLocaleString('en-US')} cached) · ${usage.output_tokens.toLocaleString('en-US')} output` : 'unavailable';
+ const range = result.estimatedCostRange;
+ const cost = range && range.max !== range.min
+ ? range.max === null ? `$${range.min.toFixed(4)} (upper estimate unavailable)` : `$${range.min.toFixed(4)}–$${range.max.toFixed(4)}`
+ : result.estimatedCost === undefined ? 'unavailable' : `$${result.estimatedCost.toFixed(4)}`;
const parts = [
`**Scan:** ${result.scanStatus} · **Findings policy:** ${result.policyStatus} · **Report:** ${result.reportStatus}`,
`**Findings:** ${counts}`,
`**Scope:** ${inputs.scope}${inputs.paths.length ? ` <code>${esc(inputs.paths.join(', '))}</code>` : ''} · **Mode:** ${inputs.mode}`,
`**Commit:** <code>${target.scannedSha}</code>`,
- `**Model:** <code>${esc(inputs.model)}</code> · **Reasoning effort:** ${inputs.effort}`,
+ `**Configured provider:** <code>${esc(inputs.provider)}</code> · **Model:** <code>${esc(inputs.model)}</code> · **Reasoning effort:** ${inputs.effort}`,
+ `**Tokens:** ${tokens} · **Estimated cost:** ${cost}`,
`**Failure threshold:** ${inputs.failOnSeverity === 'none' ? 'report-only findings' : inputs.failOnSeverity + ' and above'}. Findings at or above the configured threshold fail the action. Scanner and required reporting errors also fail the action.`,
...(inputs.maxCost !== undefined ? [`**Stop threshold:** $${inputs.maxCost} (estimated; in-flight requests can exceed it)`] : []),
'Applicable root and nested SECURITY.md policy is discovered by the scanner.',
diff --git a/github-action/src/results.ts b/github-action/src/results.ts
index 51ccd36d4..7caac86a1 100644
--- a/github-action/src/results.ts
+++ b/github-action/src/results.ts
@@ -36,7 +36,9 @@ export interface ScanResults {
reportStatus: ReportStatus;
findings: Finding[];
counts?: Record<Severity, number>;
+ usage?: { input_tokens: number; cached_input_tokens: number; output_tokens: number };
estimatedCost?: number;
+ estimatedCostRange?: { min: number; max: number | null };
paths: ResultPaths;
errors: string[];
sarifUploadReady: boolean;
@@ -143,7 +145,10 @@ export async function analyzeResults(options: ResultOptions): Promise<ScanResult
}
result.counts = { critical: 0, high: 0, medium: 0, low: 0, informational: 0 };
for (const finding of result.findings) result.counts[finding.severity] += 1;
+ const usage = value.turn?.usage;
+ result.usage = usage == null ? undefined : {...usage, cached_input_tokens: usage.cached_input_tokens ?? 0};
result.estimatedCost = value.cost?.estimatedUsd;
+ result.estimatedCostRange = value.cost?.estimatedUsdRange;
result.paths = {resultsDirectory: options.resultsDirectory, manifestPath: join(options.resultsDirectory, 'scan-manifest.json'),
jsonPath: join(options.resultsDirectory, 'findings.json'), coveragePath: join(options.resultsDirectory, 'coverage.json'), sarifPath: ''};
const warnings: string[] = value.warnings ?? [];
diff --git a/github-action/tests/main.test.ts b/github-action/tests/main.test.ts
index 8e2e250d3..439a94554 100644
--- a/github-action/tests/main.test.ts
+++ b/github-action/tests/main.test.ts
@@ -213,6 +213,22 @@ test('scheduled complete report-only scan succeeds', async (t) => {
assert.match(result.summary, /\*\*Scan completed\. Findings are reported without failing the job\.\*\*/);
});
+test('usage without a cached count preserves a successful Action and its reports', async t => {
+ const app = await harness(t);
+ app.configure({scanResult: value => {
+ value.turn = {usage: {input_tokens: 1200, output_tokens: 50}};
+ }});
+ const result = await app.run();
+ assert.equal(result.exitCode, 0);
+ assert.equal(result.outputs['scan-status'], 'completed');
+ assert.equal(result.outputs['policy-status'], 'passed');
+ assert.equal(result.outputs['report-status'], 'ready');
+ assert.equal(result.outputs['sarif-upload-ready'], 'true');
+ assert.equal(result.cleanups, 1);
+ assert.ok(result.summary.includes('1,200 input (0 cached) · 50 output'));
+ assert.equal(result.outputs['estimated-cost'], '0.125');
+});
+
test('unrelated ambient input variables do not reject a valid scan', async (t) => {
const app = await harness(t); process.env.INPUT_FOO = 'unrelated workflow value';
app.setInput('effort', 'medium');
diff --git a/github-action/tests/reporting.test.ts b/github-action/tests/reporting.test.ts
index 6a0b476dd..efd87269a 100644
--- a/github-action/tests/reporting.test.ts
+++ b/github-action/tests/reporting.test.ts
@@ -63,3 +63,22 @@ test('report fields preserve Markdown punctuation as literal content', () => {
assert.ok(summary.includes('<code>pkg/&#95;&#95;init&#95;&#95;.py</code>'));
assert.ok(summary.includes('<code>pkg/&#95;&#95;init&#95;&#95;.py:1</code>'));
});
+
+test('summary shows configured provider, measured tokens and available cost estimates', () => {
+ const inputs = parseInputs(name => name === 'provider' ? 'amazon-bedrock' : '', '/checkout');
+ const target = {repository:'/checkout',scannedSha:'a'.repeat(40),analysisRef:'refs/heads/main',publishable:true,emptyDiff:false};
+ const result = {scanStatus:'completed',policyStatus:'passed',reportStatus:'ready',errors:[],findings:[]} as unknown as ScanResults;
+ const usage = {input_tokens:1200,cached_input_tokens:200,output_tokens:50};
+ for (const [metrics, expected] of [
+ [{usage, estimatedCost:0.125}, '1,200 input (200 cached) · 50 output · **Estimated cost:** $0.1250'],
+ [{usage, estimatedCost:0.125, estimatedCostRange:{min:0.125,max:0.25}}, '1,200 input (200 cached) · 50 output · **Estimated cost:** $0.1250–$0.2500'],
+ [{usage, estimatedCost:0.125, estimatedCostRange:{min:0.125,max:null}}, '1,200 input (200 cached) · 50 output · **Estimated cost:** $0.1250 (upper estimate unavailable)'],
+ [{usage}, '1,200 input (200 cached) · 50 output · **Estimated cost:** unavailable'],
+ [{usage:{input_tokens:0,cached_input_tokens:0,output_tokens:0},estimatedCost:0,estimatedCostRange:{min:0,max:0}}, '0 input (0 cached) · 0 output · **Estimated cost:** $0.0000'],
+ [{}, 'unavailable · **Estimated cost:** unavailable'],
+ ] as const) {
+ const summary = resultSummary({...result, ...metrics}, inputs, target);
+ assert.ok(summary.includes('**Configured provider:** <code>amazon-bedrock</code>'));
+ assert.ok(summary.includes(`**Tokens:** ${expected}`));
+ }
+});
diff --git a/github-action/tests/results.test.ts b/github-action/tests/results.test.ts
index 973e4103f..b2aeb8fe4 100644
--- a/github-action/tests/results.test.ts
+++ b/github-action/tests/results.test.ts
@@ -21,7 +21,8 @@ async function fixture(t: { after(fn: () => Promise<void>): void }): Promise<Res
coverage: await document('coverage.json'),
scanDir: root,
sarifPath: join(root, 'exports/results.sarif'),
- cost: { estimatedUsd: 0.125 },
+ turn: { usage: { input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 } },
+ cost: { estimatedUsd: 0.125, estimatedUsdRange: { min: 0.125, max: 0.25, context: 'unknown' } },
}),
};
}
@@ -40,12 +41,27 @@ test('adapts a synthetic CLI JSON result into findings, counts and report output
assert.equal(result.reportStatus, 'ready');
assert.equal(result.sarifUploadReady, true);
assert.deepEqual(result.counts, { critical: 0, high: 1, medium: 0, low: 0, informational: 0 });
+ assert.deepEqual(result.usage, { input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 });
assert.equal(result.estimatedCost, 0.125);
+ assert.deepEqual(result.estimatedCostRange, { min: 0.125, max: 0.25, context: 'unknown' });
assert.equal(result.findings[0]?.path, 'src/extract.py');
assert.equal(result.findings[0]?.startLine, 41);
assert.equal(result.paths.jsonPath, join(opts.resultsDirectory, 'findings.json'));
});
+test('token usage remains available for unpriced models and distinguishes missing measurements from zero', async (t) => {
+ const opts = await fixture(t);
+ for (const usage of [{ input_tokens: 0, cached_input_tokens: 0, output_tokens: 0 },
+ { input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 }, null, undefined]) {
+ change(opts, value => { value.turn = { usage }; value.cost = null; });
+ const result = await analyzeResults(opts);
+ assert.equal(result.scanStatus, 'completed');
+ assert.deepEqual(result.usage, usage ?? undefined);
+ assert.equal(result.estimatedCost, undefined);
+ assert.equal(result.estimatedCostRange, undefined);
+ }
+});
+
test('uses the structured CLI findings instead of reparsing report documents', async (t) => {
const opts = await fixture(t);
change(opts, (value) => { value.findings.findings[0].title = 'Title from the CLI result'; });
diff --git a/sdk/typescript/tests-ts/azure-pipelines-example.test.ts b/sdk/typescript/tests-ts/azure-pipelines-example.test.ts
index adc42cb28..785edd9d7 100644
--- a/sdk/typescript/tests-ts/azure-pipelines-example.test.ts
+++ b/sdk/typescript/tests-ts/azure-pipelines-example.test.ts
@@ -97,6 +97,8 @@ test("runs a report-only full scan outside the target checkout", () => {
"standard",
"--effort",
"high",
+ "--codex",
+ 'responses_api_metadata.codex_security_integration="azure_pipelines"',
"--output-dir",
`${result.root}/scan`,
"--json",