OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
refactor(evals): share triage assertion output parsing (#1201)
mldangelo-oaic8066c435caf
diff --git a/evals/triage-finding/assertions/calibration-evidence.js b/evals/triage-finding/assertions/calibration-evidence.jsindex f3c535a70..dc9af63b8 100644--- a/evals/triage-finding/assertions/calibration-evidence.js+++ b/evals/triage-finding/assertions/calibration-evidence.js@@ -1,24 +1,12 @@"use strict";-function outputText(output) {- return typeof output === "string" ? output : JSON.stringify(output);-}--function parseExpectedTerms(value) {- if (Array.isArray(value)) {- return value.map(String).map((term) => term.trim()).filter(Boolean);- }- if (typeof value === "string") {- return value- .split(",")- .map((term) => term.trim())- .filter(Boolean);- }- return [];-}+const { outputText, parseExpected } = require("./output");module.exports = (output, context) => {- const expectedTerms = parseExpectedTerms(context.vars.expected_evidence_terms);+ const expectedTerms = parseExpected(+ context.vars.expected_evidence_terms,+ true,+ );if (expectedTerms.length === 0) {return {pass: true,@@ -29,11 +17,10 @@ module.exports = (output, context) => {const normalizedOutput = outputText(output).toLowerCase();const missingTerms = expectedTerms.filter((term) => !normalizedOutput.includes(term.toLowerCase()));- const foundCount = expectedTerms.length - missingTerms.length;return {pass: missingTerms.length === 0,- score: foundCount / expectedTerms.length,+ score: (expectedTerms.length - missingTerms.length) / expectedTerms.length,reason:missingTerms.length === 0? "All required calibration evidence terms were cited."diff --git a/evals/triage-finding/assertions/github-rest-intake.js b/evals/triage-finding/assertions/github-rest-intake.jsindex d7bdd9d0f..15cea1b32 100644--- a/evals/triage-finding/assertions/github-rest-intake.js+++ b/evals/triage-finding/assertions/github-rest-intake.js@@ -1,20 +1,9 @@-function textFor(output) {- return typeof output === "string" ? output : JSON.stringify(output);-}+const { outputText: textFor, hasTriageJson } = require("./output");function containsAll(text, patterns) {return patterns.every((pattern) => pattern.test(text));}-function hasTriageJson(text) {- return (- /```(?:json)?\s*[\s\S]*?```/i.test(text) ||- /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||- /["']findings["']\s*:/i.test(text) ||- /["']verdict["']\s*:/i.test(text)- );-}-function endpointPattern(path, queryParts = []) {const escapedPath = path.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");const queryPatterns = queryParts.map((part) => new RegExp(part, "i"));diff --git a/evals/triage-finding/assertions/missing-input.js b/evals/triage-finding/assertions/missing-input.jsindex 2603e3193..891e959b8 100644--- a/evals/triage-finding/assertions/missing-input.js+++ b/evals/triage-finding/assertions/missing-input.js@@ -1,6 +1,4 @@-function outputText(output) {- return typeof output === "string" ? output : JSON.stringify(output);-}+const { outputText } = require("./output");function matchedFormatNames(text) {const supportedFormats = [diff --git a/evals/triage-finding/assertions/output.js b/evals/triage-finding/assertions/output.jsnew file mode 100644index 000000000..96bdd386d--- /dev/null+++ b/evals/triage-finding/assertions/output.js@@ -0,0 +1,22 @@+function outputText(output) {+ return typeof output === "string" ? output : JSON.stringify(output);+}++function hasTriageJson(text) {+ return (+ /```(?:json)?\s*[\s\S]*?```/i.test(text) ||+ /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||+ /["']findings["']\s*:/i.test(text) ||+ /["']verdict["']\s*:/i.test(text)+ );+}++function parseExpected(value, trim = typeof value === "string") {+ if (typeof value === "string") {+ value = value.split(",");+ }+ const terms = Array.isArray(value) ? value.map(String) : [];+ return trim ? terms.flatMap((term) => term.trim() || []) : terms;+}++module.exports = { outputText, hasTriageJson, parseExpected };diff --git a/evals/triage-finding/assertions/ticket-intake.js b/evals/triage-finding/assertions/ticket-intake.jsindex 7dc80eccd..7a85e8471 100644--- a/evals/triage-finding/assertions/ticket-intake.js+++ b/evals/triage-finding/assertions/ticket-intake.js@@ -1,15 +1,4 @@-function outputText(output) {- return typeof output === "string" ? output : JSON.stringify(output);-}--function hasTriageJson(text) {- return (- /```(?:json)?\s*[\s\S]*?```/i.test(text) ||- /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||- /["']findings["']\s*:/i.test(text) ||- /["']verdict["']\s*:/i.test(text)- );-}+const { outputText, hasTriageJson } = require("./output");const expectedPatterns = {unavailable: [/connector|Linear/i, /connect|authenticate|reauthorize/i, /paste|provide.*content/i],diff --git a/evals/triage-finding/assertions/triage-io.js b/evals/triage-finding/assertions/triage-io.jsindex a8b944170..a3f5aa6a9 100644--- a/evals/triage-finding/assertions/triage-io.js+++ b/evals/triage-finding/assertions/triage-io.js@@ -1,15 +1,4 @@-function parseExpected(value) {- if (Array.isArray(value)) {- return value.map(String);- }- if (typeof value === "string") {- return value- .split(",")- .map((item) => item.trim())- .filter(Boolean);- }- return [];-}+const { parseExpected } = require("./output");function expectedScanbenchMapping(context) {const caseId = String(context.vars.case_id || "");