Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(evals): reuse parsing and Git helpers (#1222)

mldangelo-oaidcf3ed53d40d
diff --git a/evals/triage-finding/assertions/triage-io.js b/evals/triage-finding/assertions/triage-io.js
index a3f5aa6a9..797ccfd59 100644
--- a/evals/triage-finding/assertions/triage-io.js
+++ b/evals/triage-finding/assertions/triage-io.js
@@ -1,4 +1,5 @@
const { parseExpected } = require("./output");
+const { extractTriageResult } = require("../sastbench/scripts/sastbench-result");
function expectedScanbenchMapping(context) {
const caseId = String(context.vars.case_id || "");
@@ -36,32 +37,11 @@ function expectedScanbenchMapping(context) {
return mappings;
}
-function extractJson(output) {
- const text = typeof output === "string" ? output : JSON.stringify(output);
- const fencedBlocks = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)].map((match) =>
- match[1].trim(),
- );
- const candidates = fencedBlocks.length > 0 ? fencedBlocks : [text.slice(text.indexOf("{"), text.lastIndexOf("}") + 1)];
-
- for (const candidate of candidates) {
- if (!candidate) {
- continue;
- }
- try {
- const parsed = JSON.parse(candidate);
- if (parsed && parsed.schema_version === "triage-finding/v0") {
- return parsed;
- }
- } catch {
- // Keep trying other candidates.
- }
- }
-
- throw new Error("Could not find a parseable triage-finding/v0 JSON block.");
-}
-
module.exports = (output, context) => {
- const result = extractJson(output);
+ const result = extractTriageResult(
+ output,
+ "Could not find a parseable triage-finding/v0 JSON block.",
+ );
const expectedIds = parseExpected(context.vars.expected_ids);
const expectedSourceTypes = parseExpected(context.vars.expected_source_types);
const expectedVerdicts = parseExpected(context.vars.expected_verdicts);
diff --git a/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.js b/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.js
index a1d2e3c30..ab99177e8 100644
--- a/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.js
+++ b/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.js
@@ -179,6 +179,7 @@ if (require.main === module) {
module.exports = {
runGit,
+ gitOutput,
DEFAULT_CACHE_ROOT,
parseArgs,
repositoryCacheId,
diff --git a/evals/triage-finding/sastbench/scripts/install-sastbench.js b/evals/triage-finding/sastbench/scripts/install-sastbench.js
index 50a402fb3..7889539ac 100644
--- a/evals/triage-finding/sastbench/scripts/install-sastbench.js
+++ b/evals/triage-finding/sastbench/scripts/install-sastbench.js
@@ -1,7 +1,7 @@
#!/usr/bin/env node
"use strict";
-const childProcess = require("node:child_process");
+const { runGit, gitOutput } = require("./hydrate-sastbench-repos");
const fs = require("node:fs");
const path = require("node:path");
@@ -34,18 +34,6 @@ function parseArgs(argv) {
return { ...args, target: path.resolve(args.target) };
}
-function runGit(args, cwd) {
- return childProcess.execFileSync("git", args, {
- cwd,
- encoding: "utf8",
- stdio: ["ignore", "pipe", "pipe"],
- });
-}
-
-function gitOutput(args, cwd) {
- return runGit(args, cwd).trim();
-}
-
function inspectInstallation(target) {
const resolvedTarget = path.resolve(target);
if (!fs.existsSync(path.join(resolvedTarget, ".git"))) {
diff --git a/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.js b/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.js
index 2f7736751..fc6d4b060 100644
--- a/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.js
+++ b/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.js
@@ -28,7 +28,6 @@ function copyDirectory(sourceRoot, targetRoot, excludedNames = new Set()) {
if (!entry.isFile()) {
throw new Error("Refusing to stage non-file runtime entry: " + sourcePath);
}
- fs.mkdirSync(path.dirname(targetPath), { recursive: true });
fs.copyFileSync(sourcePath, targetPath);
}
}
diff --git a/evals/triage-finding/sastbench/scripts/sastbench-result.js b/evals/triage-finding/sastbench/scripts/sastbench-result.js
index ae51bce98..00923e45c 100644
--- a/evals/triage-finding/sastbench/scripts/sastbench-result.js
+++ b/evals/triage-finding/sastbench/scripts/sastbench-result.js
@@ -3,7 +3,10 @@
const VALID_VERDICTS = new Set(["confirmed", "not_actionable", "needs_review"]);
const VALID_LABELS = new Set(["true_positive", "false_positive"]);
-function extractTriageResult(output) {
+function extractTriageResult(
+ output,
+ failureMessage = "Could not find a parseable triage-finding/v0 JSON result",
+) {
const text = typeof output === "string" ? output : JSON.stringify(output);
const fencedBlocks = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)].map((match) =>
match[1].trim(),
@@ -26,7 +29,7 @@ function extractTriageResult(output) {
// The response may contain more than one fenced block. Try the next one.
}
}
- throw new Error("Could not find a parseable triage-finding/v0 JSON result");
+ throw new Error(failureMessage);
}
function parseCaseOutcome(output, expectedInputId) {
diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-result.js b/evals/triage-finding/sastbench/scripts/test-sastbench-result.js
index 231980625..7a799e7c1 100644
--- a/evals/triage-finding/sastbench/scripts/test-sastbench-result.js
+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-result.js
@@ -31,7 +31,9 @@ assert.equal(
extractTriageResult(JSON.stringify(triageResult("not_actionable"))).findings[0].verdict,
"not_actionable",
);
-assert.throws(() => extractTriageResult("no json"), /triage-finding\/v0/);
+assert.throws(() => extractTriageResult("no json"), {
+ message: "Could not find a parseable triage-finding/v0 JSON result",
+});
assert.throws(
() => parseCaseOutcome(JSON.stringify(triageResult("confirmed", "wrong")), "sastbench-000000"),
/input_id mismatch/,
diff --git a/evals/triage-finding/scripts/test-triage-io.js b/evals/triage-finding/scripts/test-triage-io.js
index 20f20503c..18ed75cf1 100644
--- a/evals/triage-finding/scripts/test-triage-io.js
+++ b/evals/triage-finding/scripts/test-triage-io.js
@@ -4,6 +4,10 @@
const assert = require("assert");
const triageIo = require("../assertions/triage-io.js");
+assert.throws(() => triageIo("no json", {}), {
+ message: "Could not find a parseable triage-finding/v0 JSON block.",
+});
+
function outputFor({ inputId, sourceType, verdict }) {
const stackRank = {
rank_queue: verdict === "not_actionable" ? null : verdict,