OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test: include triage I/O assertions in deterministic checks (#728)
mldangelo-oai324c225839ce
diff --git a/plugins/codex-security/skills/triage-finding/evals/package.json b/plugins/codex-security/skills/triage-finding/evals/package.jsonindex c8694b5b6..9ed720c7b 100644--- a/plugins/codex-security/skills/triage-finding/evals/package.json+++ b/plugins/codex-security/skills/triage-finding/evals/package.json@@ -14,7 +14,7 @@"test:calibration-generate": "node scripts/test-generate-calibration-tests.js","test:calibration-hydrate": "node scripts/test-hydrate-calibration-repos.js","test:dataset": "node scripts/test-calibration-dataset.js",- "test:deterministic": "node scripts/test-app-surface-instructions.js && node scripts/test-calibration-dataset.js && node scripts/test-generate-calibration-tests.js && node scripts/test-calibration-evidence.js && node scripts/test-hydrate-calibration-repos.js && node sastbench/scripts/test-sastbench-lib.js && node sastbench/scripts/test-install-sastbench.js && node sastbench/scripts/test-hydrate-sastbench-repos.js && node sastbench/scripts/test-sastbench-result.js && node sastbench/scripts/test-promptfoo-native-harness.js",+ "test:deterministic": "node scripts/test-app-surface-instructions.js && node scripts/test-triage-io.js && node scripts/test-calibration-dataset.js && node scripts/test-generate-calibration-tests.js && node scripts/test-calibration-evidence.js && node scripts/test-hydrate-calibration-repos.js && node sastbench/scripts/test-sastbench-lib.js && node sastbench/scripts/test-install-sastbench.js && node sastbench/scripts/test-hydrate-sastbench-repos.js && node sastbench/scripts/test-sastbench-result.js && node sastbench/scripts/test-promptfoo-native-harness.js","test:instructions": "node scripts/test-app-surface-instructions.js","test:sastbench": "node sastbench/scripts/test-sastbench-lib.js && node sastbench/scripts/test-install-sastbench.js && node sastbench/scripts/test-hydrate-sastbench-repos.js && node sastbench/scripts/test-sastbench-result.js && node sastbench/scripts/test-promptfoo-native-harness.js","validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./promptfooconfig.yaml",diff --git a/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.js b/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.jsindex a4f26e105..20f20503c 100644--- a/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.js+++ b/plugins/codex-security/skills/triage-finding/evals/scripts/test-triage-io.js@@ -5,6 +5,12 @@ const assert = require("assert");const triageIo = require("../assertions/triage-io.js");function outputFor({ inputId, sourceType, verdict }) {+ const stackRank = {+ rank_queue: verdict === "not_actionable" ? null : verdict,+ rank: verdict === "not_actionable" ? null : 1,+ rationale: verdict === "not_actionable" ? "not actionable" : "Example ranking",+ drivers: [],+ };return `\`\`\`json{"schema_version": "triage-finding/v0",@@ -21,6 +27,7 @@ function outputFor({ inputId, sourceType, verdict }) {"normalized_input": {},"verdict": "${verdict}","confidence": "high",+ "exploitability_stack_rank": ${JSON.stringify(stackRank)},"affected_locations": [],"reachable_path": [],"evidence": [],