Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(evals): share triage assertion output parsing (#1201)

mldangelo-oaic8066c435caf
diff --git a/evals/triage-finding/assertions/calibration-evidence.js b/evals/triage-finding/assertions/calibration-evidence.js
index f3c535a70..dc9af63b8 100644
--- a/evals/triage-finding/assertions/calibration-evidence.js
+++ b/evals/triage-finding/assertions/calibration-evidence.js
@@ -1,24 +1,12 @@
"use strict";
-function outputText(output) {
- return typeof output === "string" ? output : JSON.stringify(output);
-}
-
-function parseExpectedTerms(value) {
- if (Array.isArray(value)) {
- return value.map(String).map((term) => term.trim()).filter(Boolean);
- }
- if (typeof value === "string") {
- return value
- .split(",")
- .map((term) => term.trim())
- .filter(Boolean);
- }
- return [];
-}
+const { outputText, parseExpected } = require("./output");
module.exports = (output, context) => {
- const expectedTerms = parseExpectedTerms(context.vars.expected_evidence_terms);
+ const expectedTerms = parseExpected(
+ context.vars.expected_evidence_terms,
+ true,
+ );
if (expectedTerms.length === 0) {
return {
pass: true,
@@ -29,11 +17,10 @@ module.exports = (output, context) => {
const normalizedOutput = outputText(output).toLowerCase();
const missingTerms = expectedTerms.filter((term) => !normalizedOutput.includes(term.toLowerCase()));
- const foundCount = expectedTerms.length - missingTerms.length;
return {
pass: missingTerms.length === 0,
- score: foundCount / expectedTerms.length,
+ score: (expectedTerms.length - missingTerms.length) / expectedTerms.length,
reason:
missingTerms.length === 0
? "All required calibration evidence terms were cited."
diff --git a/evals/triage-finding/assertions/github-rest-intake.js b/evals/triage-finding/assertions/github-rest-intake.js
index d7bdd9d0f..15cea1b32 100644
--- a/evals/triage-finding/assertions/github-rest-intake.js
+++ b/evals/triage-finding/assertions/github-rest-intake.js
@@ -1,20 +1,9 @@
-function textFor(output) {
- return typeof output === "string" ? output : JSON.stringify(output);
-}
+const { outputText: textFor, hasTriageJson } = require("./output");
function containsAll(text, patterns) {
return patterns.every((pattern) => pattern.test(text));
}
-function hasTriageJson(text) {
- return (
- /```(?:json)?\s*[\s\S]*?```/i.test(text) ||
- /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||
- /["']findings["']\s*:/i.test(text) ||
- /["']verdict["']\s*:/i.test(text)
- );
-}
-
function endpointPattern(path, queryParts = []) {
const escapedPath = path.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const queryPatterns = queryParts.map((part) => new RegExp(part, "i"));
diff --git a/evals/triage-finding/assertions/missing-input.js b/evals/triage-finding/assertions/missing-input.js
index 2603e3193..891e959b8 100644
--- a/evals/triage-finding/assertions/missing-input.js
+++ b/evals/triage-finding/assertions/missing-input.js
@@ -1,6 +1,4 @@
-function outputText(output) {
- return typeof output === "string" ? output : JSON.stringify(output);
-}
+const { outputText } = require("./output");
function matchedFormatNames(text) {
const supportedFormats = [
diff --git a/evals/triage-finding/assertions/output.js b/evals/triage-finding/assertions/output.js
new file mode 100644
index 000000000..96bdd386d
--- /dev/null
+++ b/evals/triage-finding/assertions/output.js
@@ -0,0 +1,22 @@
+function outputText(output) {
+ return typeof output === "string" ? output : JSON.stringify(output);
+}
+
+function hasTriageJson(text) {
+ return (
+ /```(?:json)?\s*[\s\S]*?```/i.test(text) ||
+ /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||
+ /["']findings["']\s*:/i.test(text) ||
+ /["']verdict["']\s*:/i.test(text)
+ );
+}
+
+function parseExpected(value, trim = typeof value === "string") {
+ if (typeof value === "string") {
+ value = value.split(",");
+ }
+ const terms = Array.isArray(value) ? value.map(String) : [];
+ return trim ? terms.flatMap((term) => term.trim() || []) : terms;
+}
+
+module.exports = { outputText, hasTriageJson, parseExpected };
diff --git a/evals/triage-finding/assertions/ticket-intake.js b/evals/triage-finding/assertions/ticket-intake.js
index 7dc80eccd..7a85e8471 100644
--- a/evals/triage-finding/assertions/ticket-intake.js
+++ b/evals/triage-finding/assertions/ticket-intake.js
@@ -1,15 +1,4 @@
-function outputText(output) {
- return typeof output === "string" ? output : JSON.stringify(output);
-}
-
-function hasTriageJson(text) {
- return (
- /```(?:json)?\s*[\s\S]*?```/i.test(text) ||
- /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||
- /["']findings["']\s*:/i.test(text) ||
- /["']verdict["']\s*:/i.test(text)
- );
-}
+const { outputText, hasTriageJson } = require("./output");
const expectedPatterns = {
unavailable: [/connector|Linear/i, /connect|authenticate|reauthorize/i, /paste|provide.*content/i],
diff --git a/evals/triage-finding/assertions/triage-io.js b/evals/triage-finding/assertions/triage-io.js
index a8b944170..a3f5aa6a9 100644
--- a/evals/triage-finding/assertions/triage-io.js
+++ b/evals/triage-finding/assertions/triage-io.js
@@ -1,15 +1,4 @@
-function parseExpected(value) {
- if (Array.isArray(value)) {
- return value.map(String);
- }
- if (typeof value === "string") {
- return value
- .split(",")
- .map((item) => item.trim())
- .filter(Boolean);
- }
- return [];
-}
+const { parseExpected } = require("./output");
function expectedScanbenchMapping(context) {
const caseId = String(context.vars.case_id || "");