OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(evals): parse complete triage JSON before fenced content (#1512)
mldangelo-oai6f4c62875baa
diff --git a/evals/triage-finding/assertions/output.mts b/evals/triage-finding/assertions/output.mtsindex 88f9c0a08..29624fe9b 100644--- a/evals/triage-finding/assertions/output.mts+++ b/evals/triage-finding/assertions/output.mts@@ -70,13 +70,47 @@ export function extractTriageResult(failureMessage = "Could not find a parseable triage-finding/v0 JSON result",) {const text = outputText(output);- const fencedBlocks = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)].map(- (match) => match[1].trim(),- );+ try {+ const parsed = JSON.parse(text) as TriageResult;+ if (parsed && parsed.schema_version === "triage-finding/v0") return parsed;+ } catch {+ // The result may be surrounded by prose or Markdown fences.+ }+ const fencedBlocks: [number, string][] = [];+ const openingFence = /```(?:json)?\s*/gi;+ // Complete JSON strings may contain backticks, but never literal line breaks.+ const tokens = /"|```/g;+ const quoted = /"(?:\\.|[^"\\\r\n])*(")?/y;+ // Escaped quotes within an unterminated string cannot start complete strings.+ let unterminatedUntil = 0;+ let opening;+ while ((opening = openingFence.exec(text))) {+ const start = openingFence.lastIndex;+ tokens.lastIndex = start;+ let token;+ while ((token = tokens.exec(text)) && token[0] !== "```") {+ if (token.index < unterminatedUntil) continue;+ quoted.lastIndex = token.index;+ const string = quoted.exec(text)!;+ if (string[1]) tokens.lastIndex = quoted.lastIndex;+ else unterminatedUntil = quoted.lastIndex;+ }+ if (!token) break;+ fencedBlocks.push([opening.index, text.slice(start, token.index).trim()]);+ openingFence.lastIndex = tokens.lastIndex;+ }+ // Preserve legacy inline framing when malformed quoted text hides a later fence.+ for (const match of text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)) {+ fencedBlocks.push([match.index, match[1].trim()]);+ }const start = text.indexOf("{");const end = text.lastIndexOf("}");const candidates =- fencedBlocks.length > 0 ? fencedBlocks : [text.slice(start, end + 1)];+ fencedBlocks.length > 0+ ? fencedBlocks+ .sort((left, right) => left[0] - right[0])+ .map(([, body]) => body)+ : [text.slice(start, end + 1)];for (const candidate of candidates) {try {const parsed = JSON.parse(candidate) as TriageResult;diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mtsindex 73aa2e259..96d9eab75 100644--- a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts@@ -14,6 +14,7 @@ function triageResult(verdict: string, inputId = "sastbench-000000") {input_id: inputId,source_type: "scanner_ticket",verdict,+ evidence: ['Literal ```code``` fence, "quoted" and \\ escaped text'],},],};@@ -37,6 +38,54 @@ assert.throws(),{ message: "Could not find a parseable triage-finding/v0 JSON result" },);+const truncated =+ '```json\n{"schema_version":"triage-finding/v0","findings":[{"evidence":["' ++ '\\"x'.repeat(32_000);+assert.throws(() => extractTriageResult(truncated), {+ message: "Could not find a parseable triage-finding/v0 JSON result",+});+const truncatedMetrics = addSastBenchMetrics(+ extensionContext({+ caseId: "sastbench-000000",+ expectedGroundTruth: "true_positive",+ output: truncated,+ }),+).result;+assert.equal(truncatedMetrics.metadata.sastbench.status, "invalid_output");+assert.equal(truncatedMetrics.namedScores.invalid_output, 1);+const inlineSecond = triageResult("confirmed");+inlineSecond.findings[0].evidence = [];+for (const output of [+ JSON.stringify(triageResult("confirmed")),+ JSON.stringify(triageResult("confirmed"), null, 2),+ '```json "invalid ``` ```json ' ++ JSON.stringify(inlineSecond) ++ "```\n```json\n" ++ JSON.stringify(triageResult("not_actionable")) ++ "\n```",+ fenced.replaceAll("\n", "\r\n"),+ truncated + "\n```\n" + fenced,+ 'Prose {"other": true}\n' + fenced.replaceAll("```", " ```"),+ '```json\n"invalid\n```\n' + fenced,+ 'Prose {"other": true} ```json ' ++ JSON.stringify(triageResult("confirmed")) ++ "``` trailing prose",+]) {+ assert.equal(+ parseCaseOutcome(output, "sastbench-000000").verdict,+ "confirmed",+ );+ const result = addSastBenchMetrics(+ extensionContext({+ caseId: "sastbench-000000",+ expectedGroundTruth: "true_positive",+ output,+ }),+ ).result;+ assert.equal(result.metadata.sastbench.status, "ok");+ assert.equal(result.namedScores.strict_tp, 1);+ assert.equal(result.namedScores.strict_fn, 0);+}assert.throws(() =>parseCaseOutcome(diff --git a/evals/triage-finding/scripts/test-triage-io.mts b/evals/triage-finding/scripts/test-triage-io.mtsindex 9af46dc92..14be2c5f7 100644--- a/evals/triage-finding/scripts/test-triage-io.mts+++ b/evals/triage-finding/scripts/test-triage-io.mts@@ -83,7 +83,7 @@ assertPasses(outputFor({inputId: "input-001",verdict: "confirmed",- }),+ }).replace('"evidence": []', '"evidence": ["Inline ``` fence in source."]'),{vars: {case_id: "case-001",