OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtemp, readFile, writeFile, rm, realpath, symlink, link } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { analyzeResults, readReportFile, type ResultOptions } from '../src/results.ts';
import { collectReports } from '../src/artifacts.ts';
import { copyCompletedScan } from './fixtures.ts';
async function fixture(t: { after(fn: () => Promise<void>): void }): Promise<ResultOptions> {
const root = await realpath(await mkdtemp(join(tmpdir(), 'codex-results-test-')));
t.after(() => rm(root, { recursive: true, force: true }));
await copyCompletedScan(root);
const document = async (name: string) => JSON.parse(await readFile(join(root, name), 'utf8'));
return {
resultsDirectory: root, scannedSha: 'a'.repeat(40), exitCode: 0, publishable: true,
stdout: JSON.stringify({
manifest: await document('scan-manifest.json'),
findings: await document('findings.json'),
coverage: await document('coverage.json'),
scanDir: root,
sarifPath: join(root, 'exports/results.sarif'),
turn: { usage: { input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 } },
cost: { estimatedUsd: 0.125, estimatedUsdRange: { min: 0.125, max: 0.25, context: 'unknown' } },
}),
};
}
function change(options: ResultOptions, update: (value: any) => void): void {
const value = JSON.parse(options.stdout);
update(value);
options.stdout = JSON.stringify(value);
}
test('adapts a synthetic CLI JSON result into findings, counts and report outputs', async (t) => {
const opts = await fixture(t);
change(opts, (value) => { value.findings.findings[0].title = 'Title from the CLI result'; });
const result = await analyzeResults(opts);
assert.deepEqual(result.errors, []);
assert.equal(result.scanStatus, 'completed');
assert.equal(result.policyStatus, 'passed');
assert.equal(result.reportStatus, 'ready');
assert.equal(result.sarifUploadReady, true);
assert.deepEqual(result.counts, { critical: 0, high: 1, medium: 0, low: 0, informational: 0 });
assert.deepEqual(result.usage, { input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 });
assert.equal(result.estimatedCost, 0.125);
assert.deepEqual(result.estimatedCostRange, { min: 0.125, max: 0.25, context: 'unknown' });
assert.equal(result.findings[0]?.title, 'Title from the CLI result');
assert.equal(result.findings[0]?.path, 'src/extract.py');
assert.equal(result.findings[0]?.startLine, 41);
assert.equal(result.paths.jsonPath, join(opts.resultsDirectory, 'findings.json'));
});
test('token usage remains available for unpriced models and distinguishes missing measurements from zero', async (t) => {
const opts = await fixture(t);
for (const usage of [{ input_tokens: 0, cached_input_tokens: 0, output_tokens: 0 },
{ input_tokens: 1200, cached_input_tokens: 200, output_tokens: 50 }, null, undefined]) {
change(opts, value => { value.turn = { usage }; value.cost = null; });
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'completed');
assert.deepEqual(result.usage, usage ?? undefined);
assert.equal(result.estimatedCost, undefined);
assert.equal(result.estimatedCostRange, undefined);
}
});
test('annotations prefer the root control over an earlier entry point', async (t) => {
const opts = await fixture(t);
change(opts, (value) => {
value.findings.findings[0].locations = [
{role: 'entrypoint', path: 'src/route.ts', startLine: 7},
{role: 'root_control', path: 'src/validation.ts', startLine: 30, endLine: 32},
];
});
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'completed');
assert.equal(result.findings[0]?.path, 'src/validation.ts');
assert.equal(result.findings[0]?.startLine, 30);
assert.equal(result.findings[0]?.endLine, 32);
});
test('reports must describe the requested repository or diff revision', async (t) => {
const opts = await fixture(t);
for (const kind of ['git_revision', 'git_diff']) {
change(opts, (value) => {
Object.assign(value.manifest.scan.target, {kind, revision: 'b'.repeat(40), headRevision: 'b'.repeat(40)});
});
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'failed');
assert.equal(result.sarifUploadReady, false);
assert.equal(result.paths.jsonPath, '');
assert.match(result.errors.join('\n'), /revision does not match/);
change(opts, (value) => {
value.manifest.scan.target[kind === 'git_diff' ? 'headRevision' : 'revision'] = opts.scannedSha;
});
assert.equal((await analyzeResults(opts)).scanStatus, 'completed');
}
change(opts, (value) => { Object.assign(value.manifest.scan.target, {kind: 'git_worktree', revision: opts.scannedSha}); });
assert.equal((await analyzeResults(opts)).scanStatus, 'failed', 'a dirty worktree is not the requested immutable commit');
});
test('report collection preserves credential-shaped diagnostic content', async (t) => {
const opts = await fixture(t);
const report = JSON.parse(await readFile(join(opts.resultsDirectory, 'findings.json'), 'utf8'));
report.findings[0].summary = 'Synthetic diagnostic: sk-test/secret+value';
const bytes = Buffer.from(JSON.stringify(report));
await writeFile(join(opts.resultsDirectory, 'findings.json'), bytes);
const reports = await collectReports(await analyzeResults(opts));
assert.deepEqual(reports.get('findings.json'), bytes);
});
test('operational exits and signal termination cannot turn complete output into success', async (t) => {
const opts = await fixture(t);
for (const exitCode of [2, 130, 143, null]) {
const result = await analyzeResults({ ...opts, exitCode });
assert.equal(result.scanStatus, 'failed');
assert.equal(result.policyStatus, 'not-evaluated');
assert.equal(result.sarifUploadReady, false);
assert.equal(result.findings.length, 1);
assert.equal(result.counts?.high, 1);
}
});
for (const completeness of [undefined, 'complete', 'partial', 'unknown']) {
test(`coverage notes and legacy completeness ${completeness} do not change successful execution or findings policy`, async (t) => {
const opts = await fixture(t);
change(opts, (value) => {
value.coverage.completeness = completeness;
value.coverage.explicitExclusions = [{ pattern: 'assets/', reason: 'Static assets outside review scope.' }];
value.coverage.deferred = [{ id: 'unreviewed-route', reason: 'Dependency implementation unavailable.' }];
value.coverage.surfaces[0].disposition = 'needs_follow_up';
});
for (const exitCode of [0, 1]) {
const result = await analyzeResults({ ...opts, exitCode });
assert.equal(result.scanStatus, 'completed');
assert.equal(result.policyStatus, exitCode === 0 ? 'passed' : 'failed');
assert.equal(result.sarifUploadReady, true);
assert.equal(result.findings.length, 1);
assert.equal(result.paths.jsonPath, join(opts.resultsDirectory, 'findings.json'));
assert.deepEqual(result.errors, []);
}
});
}
test('failed or interrupted manifests cannot become successful from exit 0', async (t) => {
const opts = await fixture(t);
for (const status of ['failed', 'interrupted', 'canceled']) {
change(opts, (value) => {
value.manifest.scan.status = status;
});
const result = await analyzeResults({ ...opts, exitCode: 0 });
assert.equal(result.scanStatus, 'failed', status);
assert.equal(result.policyStatus, 'not-evaluated');
assert.equal(result.sarifUploadReady, false);
assert.equal(result.findings.length, 1);
}
});
test('execution failure and target-change warnings override completed output', async (t) => {
const opts = await fixture(t);
const executionFailure = await analyzeResults({ ...opts, executionFailed: true });
assert.equal(executionFailure.scanStatus, 'failed');
assert.equal(executionFailure.policyStatus, 'not-evaluated');
assert.equal(executionFailure.sarifUploadReady, false);
const warning = 'Scan target changed during execution.';
change(opts, (value) => { value.warnings = [warning]; });
const targetChange = await analyzeResults(opts);
assert.equal(targetChange.scanStatus, 'failed');
assert.equal(targetChange.policyStatus, 'not-evaluated');
assert.equal(targetChange.sarifUploadReady, false);
assert.ok(targetChange.errors.includes(warning));
});
test('a CLI failure envelope takes precedence over retained result fields', async (t) => {
const opts = await fixture(t);
change(opts, (value) => {
Object.assign(value, { status: 'failed', code: 'SCAN_FAILED', message: 'Synthetic runtime failure.' });
});
const result = await analyzeResults({ ...opts, exitCode: 2 });
assert.equal(result.scanStatus, 'failed');
assert.equal(result.policyStatus, 'not-evaluated');
assert.equal(result.sarifUploadReady, false);
assert.equal(result.paths.jsonPath, '');
assert.deepEqual(result.errors, ['Synthetic runtime failure.']);
});
test('missing SARIF leaves a completed scan with partial reports', async (t) => {
const opts = await fixture(t); await rm(join(opts.resultsDirectory, 'exports/results.sarif'));
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'completed');
assert.equal(result.reportStatus, 'partial');
assert.equal(result.sarifUploadReady, false);
});
test('non-publishable checkouts never become SARIF upload ready', async (t) => {
const opts = await fixture(t); opts.publishable = false;
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'completed');
assert.equal(result.reportStatus, 'ready');
assert.equal(result.sarifUploadReady, false);
});
test('missing, malformed and error-only CLI output fails even when report files exist', async (t) => {
const opts = await fixture(t);
for (const stdout of ['', 'not JSON', 'null', '{}', JSON.stringify({ error: 'API key rejected' })]) {
const result = await analyzeResults({ ...opts, stdout });
assert.equal(result.scanStatus, 'failed');
assert.equal(result.policyStatus, 'not-evaluated');
assert.equal(result.sarifUploadReady, false);
assert.ok(result.errors.length > 0);
assert.equal(result.counts, undefined);
}
});
test('unreadable consumed fields fail instead of publishing misleading counts', async (t) => {
const opts = await fixture(t);
for (const update of [
(value: any) => { value.findings.findings = null; },
(value: any) => { value.findings.findings[0].severity = null; },
(value: any) => { value.coverage = null; },
(value: any) => { value.manifest.scan = null; },
]) {
const invalid = { ...opts }; change(invalid, update);
const result = await analyzeResults(invalid);
assert.equal(result.scanStatus, 'failed');
assert.equal(result.policyStatus, 'not-evaluated');
assert.equal(result.sarifUploadReady, false);
assert.equal(result.counts, undefined);
}
});
test('rejects output referring to a different scan directory', async (t) => {
const opts = await fixture(t);
change(opts, (value) => { value.scanDir = join(opts.resultsDirectory, '..'); });
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'failed');
assert.equal(result.sarifUploadReady, false);
});
test('never publishes a SARIF path outside the private report directory', async (t) => {
const opts = await fixture(t);
change(opts, (value) => { value.sarifPath = join(opts.resultsDirectory, '..', 'outside.sarif'); });
const result = await analyzeResults(opts);
assert.equal(result.paths.sarifPath, '');
assert.equal(result.sarifUploadReady, false);
});
test('unsafe source locations cannot reach annotations or published results', async (t) => {
const opts = await fixture(t);
for (const path of ['../secret', '/etc/passwd', 'https://example.com/source', 'a\\b', 'C:/source.ts', 'C:source.ts']) {
change(opts, (value) => { value.findings.findings[0].locations[0].path = path; });
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'failed');
assert.equal(result.sarifUploadReady, false);
assert.equal(result.findings[0]?.path, undefined);
}
});
test('safe reader rejects symlinks, hardlinks, FIFO and paths outside the report directory', async (t) => {
const opts = await fixture(t); const path = join(opts.resultsDirectory, 'findings.json');
await rm(path); await symlink('coverage.json', path);
await assert.rejects(readReportFile(opts.resultsDirectory, 'findings.json'));
await rm(path); await link(join(opts.resultsDirectory, 'coverage.json'), path);
await assert.rejects(readReportFile(opts.resultsDirectory, 'findings.json'));
await rm(path); execFileSync('mkfifo', [path]);
await assert.rejects(readReportFile(opts.resultsDirectory, 'findings.json'));
await assert.rejects(readReportFile(opts.resultsDirectory, '../outside'));
await rm(join(opts.resultsDirectory, 'exports'), { recursive: true });
await symlink('..', join(opts.resultsDirectory, 'exports'));
await assert.rejects(readReportFile(opts.resultsDirectory, 'exports/results.sarif'));
});
test('safe reader accepts regular reports larger than the removed Action-only limit', async (t) => {
const opts = await fixture(t);
const bytes = Buffer.alloc(16 * 1024 * 1024 + 1, ' ');
await writeFile(join(opts.resultsDirectory, 'findings.json'), bytes);
assert.equal((await readReportFile(opts.resultsDirectory, 'findings.json')).length, bytes.length);
});
test('SARIF publication rejects external references and encoded traversal paths', async (t) => {
const opts = await fixture(t);
for (const sarif of [
{runs: [{results: [{relatedLocations: [{physicalLocation: {artifactLocation: {uri: '%2e%2e/secret'}}}]}]}]},
{runs: [{results: [{locations: [{physicalLocation: {artifactLocation: {uri: 'https://example.invalid/source'}}}]}]}]},
{runs: [{results: [{locations: [{physicalLocation: {artifactLocation: {uri: 'file:/etc/passwd'}}}]}]}]},
{runs: [{results: [{locations: [{physicalLocation: {artifactLocation: {uri: 'C%3A/source.ts'}}}]}]}]},
{runs: [{externalPropertyFileReferences: {}}]},
]) {
await writeFile(join(opts.resultsDirectory, 'exports/results.sarif'), JSON.stringify(sarif));
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'completed');
assert.equal(result.reportStatus, 'partial');
assert.equal(result.paths.sarifPath, '');
assert.equal(result.sarifUploadReady, false);
}
});
test('valid empty findings have zero counts', async (t) => {
const opts = await fixture(t);
change(opts, (value) => { value.findings.findings = []; });
assert.deepEqual((await analyzeResults(opts)).counts, {critical: 0, high: 0, medium: 0, low: 0, informational: 0});
});
test('accepted relative path spellings preserve completed findings and SARIF', async (t) => {
const opts = await fixture(t);
for (const path of ['./src/extract.py', 'src//extract.py', 'module:handler.ts']) {
change(opts, (value) => { value.findings.findings[0].locations[0].path = path; });
await writeFile(join(opts.resultsDirectory, 'exports/results.sarif'), JSON.stringify({runs: [{results: [
{locations: [{physicalLocation: {artifactLocation: {uri: path.replaceAll(':', '%3A')}}}]},
]}]}));
const result = await analyzeResults(opts);
assert.equal(result.scanStatus, 'completed');
assert.equal(result.findings[0]?.path, path);
assert.equal(result.counts?.high, 1);
assert.equal(result.sarifUploadReady, true);
}
});
test('SARIF preparation removes only automationDetails and preserves the published bytes', async (t) => {
const opts = await fixture(t);
const original = JSON.parse(await readFile(join(opts.resultsDirectory, 'exports/results.sarif'), 'utf8'));
assert.ok(original.runs[0].automationDetails.id);
const result = await analyzeResults(opts);
const sarif = JSON.parse(await readFile(result.paths.sarifPath, 'utf8'));
const expected = structuredClone(original);
for (const run of expected.runs) delete run.automationDetails;
assert.deepEqual(sarif, expected);
assert.deepEqual((await collectReports(result)).get('exports/results.sarif'), await readFile(result.paths.sarifPath));
assert.equal((await analyzeResults(opts)).sarifUploadReady, true, 'preparation is repeatable');
});
test('SARIF preparation never rewrites linked report files', async (t) => {
const opts = await fixture(t);
const path = join(opts.resultsDirectory, 'exports/results.sarif');
const target = join(opts.resultsDirectory, 'findings.json');
const before = await readFile(target);
await rm(path); await symlink('../findings.json', path);
assert.equal((await analyzeResults(opts)).sarifUploadReady, false);
assert.deepEqual(await readFile(target), before);
await rm(path); await link(target, path);
assert.equal((await analyzeResults(opts)).sarifUploadReady, false);
assert.deepEqual(await readFile(target), before);
});
for (const status of ['completed', 'failed'] as const) {
test(`reads scan status ${status} without a coverage file`, async (t) => {
const options = await fixture(t)
await rm(join(options.resultsDirectory, 'coverage.json'))
change(options, (value) => {
delete value.coverage
delete value.manifest.scan.coverageRef
value.manifest.scan.status = status
value.review = value.manifest.scan.review = { excludedPaths: [] }
})
const result = await analyzeResults(options)
assert.equal(result.scanStatus, status)
assert.equal(result.paths.coveragePath, '')
assert.equal(result.paths.manifestPath, join(options.resultsDirectory, 'scan-manifest.json'))
assert.equal(result.errors.length, status === 'completed' ? 0 : 1)
});
}