Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix: set SARIF success from coverage (#601)

ianw-oai464fe75cdcb3
diff --git a/sdk/typescript/_bundled_plugin/references/sarif-adapter.md b/sdk/typescript/_bundled_plugin/references/sarif-adapter.md
index 9df8c5e7c..57feb334d 100644
--- a/sdk/typescript/_bundled_plugin/references/sarif-adapter.md
+++ b/sdk/typescript/_bundled_plugin/references/sarif-adapter.md
@@ -20,6 +20,8 @@ The adapter:
Lifecycle, rich validation evidence, attack-path context, and coverage are lossy or omitted in SARIF. Preserve them in semantic JSON.
+SARIF `executionSuccessful` is true only for complete coverage. Incomplete scans retain findings and warnings. Integrations must read the coverage file referenced by `scan.coverageRef`; manifest status `completed` alone does not mean success.
+
Automatic SARIF export during finalization is best-effort so projection errors cannot invalidate a canonical seal. Use the strict adapter entry point when a consumer requires SARIF and should surface export errors.
References:
diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py
index a9daf4d75..d99436dbf 100644
--- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py
+++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py
@@ -2290,19 +2290,17 @@ def build_sarif_projection(
raise ContractError("source root: expected an existing directory")
manifest, findings, coverage, _ = _read_sealed_scan(scan_dir, schema_dir, "SARIF projection")
sarif = build_sarif(manifest, findings, source_root)
- if coverage["completeness"] != "complete":
- run = sarif["runs"][0]
- run["properties"]["codexSecurityCoverageCompleteness"] = coverage["completeness"]
- if coverage["deferred"]:
- run["invocations"] = [
- {
- "executionSuccessful": True,
- "toolExecutionNotifications": [
- {"level": "warning", "message": {"text": item["reason"]}}
- for item in coverage["deferred"]
- ],
- }
- ]
+ run = sarif["runs"][0]
+ completeness = coverage["completeness"]
+ run["invocations"] = [{"executionSuccessful": completeness == "complete"}]
+ if completeness != "complete":
+ run["properties"]["codexSecurityCoverageCompleteness"] = completeness
+ reasons = [item["reason"] for item in coverage["deferred"]] or [
+ f"Scan coverage is {completeness}; results may be incomplete."
+ ]
+ run["invocations"][0]["toolExecutionNotifications"] = [
+ {"level": "warning", "message": {"text": reason}} for reason in reasons
+ ]
_validate_sarif(sarif)
return sarif
diff --git a/sdk/typescript/tests-ts/cli-export.test.ts b/sdk/typescript/tests-ts/cli-export.test.ts
index 37e7f6bec..78dec14f3 100644
--- a/sdk/typescript/tests-ts/cli-export.test.ts
+++ b/sdk/typescript/tests-ts/cli-export.test.ts
@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
import {
chmod,
cp,
@@ -16,7 +17,11 @@ import { join } from "node:path";
import { Writable } from "node:stream";
import { describe, expect, test } from "bun:test";
import { exportEnvironment, main } from "../src/cli.js";
-import { CodexSecurityError } from "../src/index.js";
+import {
+ CodexSecurityError,
+ type CoverageDocument,
+ type ScanManifest,
+} from "../src/index.js";
import {
SYNTHETIC_CREDENTIALS,
capture,
@@ -338,15 +343,43 @@ describe("CLI", () => {
30_000,
);
- test("writes exported findings to the requested file", async () => {
+ test.each([
+ ["complete", false],
+ ["partial", true],
+ ["unknown", true],
+ ["partial", false],
+ ["unknown", false],
+ ] as const)("exports %s, deferred: %j", async (completeness, hasDeferred) => {
const directory = await mkdtemp(join(tmpdir(), "codex-security-export-"));
try {
const scan = await copyCompletedScan(directory);
+ const manifestPath = join(scan, "scan-manifest.json");
+ const manifest = JSON.parse(
+ await readFile(manifestPath, "utf8"),
+ ) as ScanManifest;
+ const coveragePath = join(scan, manifest.scan.coverageRef);
+ const coverage = JSON.parse(
+ await readFile(coveragePath, "utf8"),
+ ) as CoverageDocument;
+ coverage.completeness = completeness;
+ coverage.deferred = hasDeferred
+ ? [
+ { id: "review", reason: "Source review did not finish." },
+ { id: "validation", reason: "Validation did not finish." },
+ ]
+ : [];
+ const coverageBytes = JSON.stringify(coverage);
+ await writeFile(coveragePath, coverageBytes);
+ manifest.scan.artifacts.find(
+ (artifact) => artifact.path === manifest.scan.coverageRef,
+ )!.sha256 = createHash("sha256").update(coverageBytes).digest("hex");
+ await writeFile(manifestPath, JSON.stringify(manifest));
+
const paths = [
- "scan-manifest.json",
- "findings.json",
- "coverage.json",
- ].map((name) => join(scan, name));
+ manifestPath,
+ join(scan, manifest.scan.findingsRef),
+ coveragePath,
+ ];
const before = await Promise.all(paths.map((path) => readFile(path)));
const source = JSON.parse(before[1]!.toString()).findings[0];
for (const [format, filename] of [
@@ -375,6 +408,27 @@ describe("CLI", () => {
const sarif = JSON.parse(contents);
expect(sarif.version).toBe("2.1.0");
const run = sarif.runs[0];
+ expect(run.invocations).toHaveLength(1);
+ const invocation = run.invocations[0];
+ expect(invocation.executionSuccessful).toBe(
+ completeness === "complete",
+ );
+ expect(run.properties.codexSecurityCoverageCompleteness).toBe(
+ completeness === "complete" ? undefined : completeness,
+ );
+ if (completeness === "complete") {
+ expect(invocation.toolExecutionNotifications).toBeUndefined();
+ } else {
+ const reasons = hasDeferred
+ ? coverage.deferred.map((item) => item.reason)
+ : [expect.stringContaining(completeness)];
+ expect(invocation.toolExecutionNotifications).toEqual(
+ reasons.map((text) => ({
+ level: "warning",
+ message: { text },
+ })),
+ );
+ }
expect(run.tool.driver.rules[0]).toMatchObject({
id: source.ruleId,
help: { markdown: expect.stringContaining(source.remediation) },
@@ -386,6 +440,7 @@ describe("CLI", () => {
]),
},
});
+ expect(run.results).toHaveLength(1);
expect(run.results[0]).toMatchObject({
ruleId: source.ruleId,
message: { text: expect.stringContaining(source.remediation) },
@@ -399,6 +454,7 @@ describe("CLI", () => {
expect(stdout.text()).toBe("");
expect(stderr.text()).toBe(`${format.toUpperCase()}: ${output}\n`);
}
+ expect(manifest.scan.status).toBe("completed");
expect(await Promise.all(paths.map((path) => readFile(path)))).toEqual(
before,
);
diff --git a/sdk/typescript/tests-ts/scan-recovery.test.ts b/sdk/typescript/tests-ts/scan-recovery.test.ts
index c731e9f94..e2c184ae6 100644
--- a/sdk/typescript/tests-ts/scan-recovery.test.ts
+++ b/sdk/typescript/tests-ts/scan-recovery.test.ts
@@ -1063,7 +1063,7 @@ describe("malformed scan artifact recovery", () => {
);
expect(sarif.runs[0]?.invocations).toEqual([
{
- executionSuccessful: true,
+ executionSuccessful: false,
toolExecutionNotifications: [
{ level: "warning", message: { text: completed.warnings[0]! } },
],