Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: include triage I/O assertions in deterministic checks (#728)

mldangelo-oai324c225839ce
diff --git a/plugins/codex-security/skills/triage-finding/evals/package.json b/plugins/codex-security/skills/triage-finding/evals/package.json
index c8694b5b6..9ed720c7b 100644
--- a/plugins/codex-security/skills/triage-finding/evals/package.json
+++ b/plugins/codex-security/skills/triage-finding/evals/package.json
@@ -14,7 +14,7 @@
"test:calibration-generate": "node scripts/test-generate-calibration-tests.js",
"test:calibration-hydrate": "node scripts/test-hydrate-calibration-repos.js",
"test:dataset": "node scripts/test-calibration-dataset.js",
- "test:deterministic": "node scripts/test-app-surface-instructions.js && node scripts/test-calibration-dataset.js && node scripts/test-generate-calibration-tests.js && node scripts/test-calibration-evidence.js && node scripts/test-hydrate-calibration-repos.js && node sastbench/scripts/test-sastbench-lib.js && node sastbench/scripts/test-install-sastbench.js && node sastbench/scripts/test-hydrate-sastbench-repos.js && node sastbench/scripts/test-sastbench-result.js && node sastbench/scripts/test-promptfoo-native-harness.js",
+ "test:deterministic": "node scripts/test-app-surface-instructions.js && node scripts/test-triage-io.js && node scripts/test-calibration-dataset.js && node scripts/test-generate-calibration-tests.js && node scripts/test-calibration-evidence.js && node scripts/test-hydrate-calibration-repos.js && node sastbench/scripts/test-sastbench-lib.js && node sastbench/scripts/test-install-sastbench.js && node sastbench/scripts/test-hydrate-sastbench-repos.js && node sastbench/scripts/test-sastbench-result.js && node sastbench/scripts/test-promptfoo-native-harness.js",
"test:instructions": "node scripts/test-app-surface-instructions.js",
"test:sastbench": "node sastbench/scripts/test-sastbench-lib.js && node sastbench/scripts/test-install-sastbench.js && node sastbench/scripts/test-hydrate-sastbench-repos.js && node sastbench/scripts/test-sastbench-result.js && node sastbench/scripts/test-promptfoo-native-harness.js",
"validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./promptfooconfig.yaml",
diff --git a/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.js b/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.js
index a4f26e105..20f20503c 100644
--- a/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.js
+++ b/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.js
@@ -5,6 +5,12 @@ const assert = require("assert");
const triageIo = require("../assertions/triage-io.js");
function outputFor({ inputId, sourceType, verdict }) {
+ const stackRank = {
+ rank_queue: verdict === "not_actionable" ? null : verdict,
+ rank: verdict === "not_actionable" ? null : 1,
+ rationale: verdict === "not_actionable" ? "not actionable" : "Example ranking",
+ drivers: [],
+ };
return `\`\`\`json
{
"schema_version": "triage-finding/v0",
@@ -21,6 +27,7 @@ function outputFor({ inputId, sourceType, verdict }) {
"normalized_input": {},
"verdict": "${verdict}",
"confidence": "high",
+ "exploitability_stack_rank": ${JSON.stringify(stackRank)},
"affected_locations": [],
"reachable_path": [],
"evidence": [],