OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
refactor(evals): reuse parsing and Git helpers (#1222)
mldangelo-oaidcf3ed53d40d
diff --git a/evals/triage-finding/assertions/triage-io.js b/evals/triage-finding/assertions/triage-io.jsindex a3f5aa6a9..797ccfd59 100644--- a/evals/triage-finding/assertions/triage-io.js+++ b/evals/triage-finding/assertions/triage-io.js@@ -1,4 +1,5 @@const { parseExpected } = require("./output");+const { extractTriageResult } = require("../sastbench/scripts/sastbench-result");function expectedScanbenchMapping(context) {const caseId = String(context.vars.case_id || "");@@ -36,32 +37,11 @@ function expectedScanbenchMapping(context) {return mappings;}-function extractJson(output) {- const text = typeof output === "string" ? output : JSON.stringify(output);- const fencedBlocks = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)].map((match) =>- match[1].trim(),- );- const candidates = fencedBlocks.length > 0 ? fencedBlocks : [text.slice(text.indexOf("{"), text.lastIndexOf("}") + 1)];-- for (const candidate of candidates) {- if (!candidate) {- continue;- }- try {- const parsed = JSON.parse(candidate);- if (parsed && parsed.schema_version === "triage-finding/v0") {- return parsed;- }- } catch {- // Keep trying other candidates.- }- }-- throw new Error("Could not find a parseable triage-finding/v0 JSON block.");-}-module.exports = (output, context) => {- const result = extractJson(output);+ const result = extractTriageResult(+ output,+ "Could not find a parseable triage-finding/v0 JSON block.",+ );const expectedIds = parseExpected(context.vars.expected_ids);const expectedSourceTypes = parseExpected(context.vars.expected_source_types);const expectedVerdicts = parseExpected(context.vars.expected_verdicts);diff --git a/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.js b/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.jsindex a1d2e3c30..ab99177e8 100644--- a/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.js+++ b/evals/triage-finding/sastbench/scripts/hydrate-sastbench-repos.js@@ -179,6 +179,7 @@ if (require.main === module) {module.exports = {runGit,+ gitOutput,DEFAULT_CACHE_ROOT,parseArgs,repositoryCacheId,diff --git a/evals/triage-finding/sastbench/scripts/install-sastbench.js b/evals/triage-finding/sastbench/scripts/install-sastbench.jsindex 50a402fb3..7889539ac 100644--- a/evals/triage-finding/sastbench/scripts/install-sastbench.js+++ b/evals/triage-finding/sastbench/scripts/install-sastbench.js@@ -1,7 +1,7 @@#!/usr/bin/env node"use strict";-const childProcess = require("node:child_process");+const { runGit, gitOutput } = require("./hydrate-sastbench-repos");const fs = require("node:fs");const path = require("node:path");@@ -34,18 +34,6 @@ function parseArgs(argv) {return { ...args, target: path.resolve(args.target) };}-function runGit(args, cwd) {- return childProcess.execFileSync("git", args, {- cwd,- encoding: "utf8",- stdio: ["ignore", "pipe", "pipe"],- });-}--function gitOutput(args, cwd) {- return runGit(args, cwd).trim();-}-function inspectInstallation(target) {const resolvedTarget = path.resolve(target);if (!fs.existsSync(path.join(resolvedTarget, ".git"))) {diff --git a/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.js b/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.jsindex 2f7736751..fc6d4b060 100644--- a/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.js+++ b/evals/triage-finding/sastbench/scripts/run-sastbench-promptfoo.js@@ -28,7 +28,6 @@ function copyDirectory(sourceRoot, targetRoot, excludedNames = new Set()) {if (!entry.isFile()) {throw new Error("Refusing to stage non-file runtime entry: " + sourcePath);}- fs.mkdirSync(path.dirname(targetPath), { recursive: true });fs.copyFileSync(sourcePath, targetPath);}}diff --git a/evals/triage-finding/sastbench/scripts/sastbench-result.js b/evals/triage-finding/sastbench/scripts/sastbench-result.jsindex ae51bce98..00923e45c 100644--- a/evals/triage-finding/sastbench/scripts/sastbench-result.js+++ b/evals/triage-finding/sastbench/scripts/sastbench-result.js@@ -3,7 +3,10 @@const VALID_VERDICTS = new Set(["confirmed", "not_actionable", "needs_review"]);const VALID_LABELS = new Set(["true_positive", "false_positive"]);-function extractTriageResult(output) {+function extractTriageResult(+ output,+ failureMessage = "Could not find a parseable triage-finding/v0 JSON result",+) {const text = typeof output === "string" ? output : JSON.stringify(output);const fencedBlocks = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)].map((match) =>match[1].trim(),@@ -26,7 +29,7 @@ function extractTriageResult(output) {// The response may contain more than one fenced block. Try the next one.}}- throw new Error("Could not find a parseable triage-finding/v0 JSON result");+ throw new Error(failureMessage);}function parseCaseOutcome(output, expectedInputId) {diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-result.js b/evals/triage-finding/sastbench/scripts/test-sastbench-result.jsindex 231980625..7a799e7c1 100644--- a/evals/triage-finding/sastbench/scripts/test-sastbench-result.js+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-result.js@@ -31,7 +31,9 @@ assert.equal(extractTriageResult(JSON.stringify(triageResult("not_actionable"))).findings[0].verdict,"not_actionable",);-assert.throws(() => extractTriageResult("no json"), /triage-finding\/v0/);+assert.throws(() => extractTriageResult("no json"), {+ message: "Could not find a parseable triage-finding/v0 JSON result",+});assert.throws(() => parseCaseOutcome(JSON.stringify(triageResult("confirmed", "wrong")), "sastbench-000000"),/input_id mismatch/,diff --git a/evals/triage-finding/scripts/test-triage-io.js b/evals/triage-finding/scripts/test-triage-io.jsindex 20f20503c..18ed75cf1 100644--- a/evals/triage-finding/scripts/test-triage-io.js+++ b/evals/triage-finding/scripts/test-triage-io.js@@ -4,6 +4,10 @@const assert = require("assert");const triageIo = require("../assertions/triage-io.js");+assert.throws(() => triageIo("no json", {}), {+ message: "Could not find a parseable triage-finding/v0 JSON block.",+});+function outputFor({ inputId, sourceType, verdict }) {const stackRank = {rank_queue: verdict === "not_actionable" ? null : verdict,