Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(evals): parse complete triage JSON before fenced content (#1512)

mldangelo-oai6f4c62875baa
diff --git a/evals/triage-finding/assertions/output.mts b/evals/triage-finding/assertions/output.mts
index 88f9c0a08..29624fe9b 100644
--- a/evals/triage-finding/assertions/output.mts
+++ b/evals/triage-finding/assertions/output.mts
@@ -70,13 +70,47 @@ export function extractTriageResult(
failureMessage = "Could not find a parseable triage-finding/v0 JSON result",
) {
const text = outputText(output);
- const fencedBlocks = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)].map(
- (match) => match[1].trim(),
- );
+ try {
+ const parsed = JSON.parse(text) as TriageResult;
+ if (parsed && parsed.schema_version === "triage-finding/v0") return parsed;
+ } catch {
+ // The result may be surrounded by prose or Markdown fences.
+ }
+ const fencedBlocks: [number, string][] = [];
+ const openingFence = /```(?:json)?\s*/gi;
+ // Complete JSON strings may contain backticks, but never literal line breaks.
+ const tokens = /"|```/g;
+ const quoted = /"(?:\\.|[^"\\\r\n])*(")?/y;
+ // Escaped quotes within an unterminated string cannot start complete strings.
+ let unterminatedUntil = 0;
+ let opening;
+ while ((opening = openingFence.exec(text))) {
+ const start = openingFence.lastIndex;
+ tokens.lastIndex = start;
+ let token;
+ while ((token = tokens.exec(text)) && token[0] !== "```") {
+ if (token.index < unterminatedUntil) continue;
+ quoted.lastIndex = token.index;
+ const string = quoted.exec(text)!;
+ if (string[1]) tokens.lastIndex = quoted.lastIndex;
+ else unterminatedUntil = quoted.lastIndex;
+ }
+ if (!token) break;
+ fencedBlocks.push([opening.index, text.slice(start, token.index).trim()]);
+ openingFence.lastIndex = tokens.lastIndex;
+ }
+ // Preserve legacy inline framing when malformed quoted text hides a later fence.
+ for (const match of text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)) {
+ fencedBlocks.push([match.index, match[1].trim()]);
+ }
const start = text.indexOf("{");
const end = text.lastIndexOf("}");
const candidates =
- fencedBlocks.length > 0 ? fencedBlocks : [text.slice(start, end + 1)];
+ fencedBlocks.length > 0
+ ? fencedBlocks
+ .sort((left, right) => left[0] - right[0])
+ .map(([, body]) => body)
+ : [text.slice(start, end + 1)];
for (const candidate of candidates) {
try {
const parsed = JSON.parse(candidate) as TriageResult;
diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts
index 73aa2e259..96d9eab75 100644
--- a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts
+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts
@@ -14,6 +14,7 @@ function triageResult(verdict: string, inputId = "sastbench-000000") {
input_id: inputId,
source_type: "scanner_ticket",
verdict,
+ evidence: ['Literal ```code``` fence, "quoted" and \\ escaped text'],
},
],
};
@@ -37,6 +38,54 @@ assert.throws(
),
{ message: "Could not find a parseable triage-finding/v0 JSON result" },
);
+const truncated =
+ '```json\n{"schema_version":"triage-finding/v0","findings":[{"evidence":["' +
+ '\\"x'.repeat(32_000);
+assert.throws(() => extractTriageResult(truncated), {
+ message: "Could not find a parseable triage-finding/v0 JSON result",
+});
+const truncatedMetrics = addSastBenchMetrics(
+ extensionContext({
+ caseId: "sastbench-000000",
+ expectedGroundTruth: "true_positive",
+ output: truncated,
+ }),
+).result;
+assert.equal(truncatedMetrics.metadata.sastbench.status, "invalid_output");
+assert.equal(truncatedMetrics.namedScores.invalid_output, 1);
+const inlineSecond = triageResult("confirmed");
+inlineSecond.findings[0].evidence = [];
+for (const output of [
+ JSON.stringify(triageResult("confirmed")),
+ JSON.stringify(triageResult("confirmed"), null, 2),
+ '```json "invalid ``` ```json ' +
+ JSON.stringify(inlineSecond) +
+ "```\n```json\n" +
+ JSON.stringify(triageResult("not_actionable")) +
+ "\n```",
+ fenced.replaceAll("\n", "\r\n"),
+ truncated + "\n```\n" + fenced,
+ 'Prose {"other": true}\n' + fenced.replaceAll("```", " ```"),
+ '```json\n"invalid\n```\n' + fenced,
+ 'Prose {"other": true} ```json ' +
+ JSON.stringify(triageResult("confirmed")) +
+ "``` trailing prose",
+]) {
+ assert.equal(
+ parseCaseOutcome(output, "sastbench-000000").verdict,
+ "confirmed",
+ );
+ const result = addSastBenchMetrics(
+ extensionContext({
+ caseId: "sastbench-000000",
+ expectedGroundTruth: "true_positive",
+ output,
+ }),
+ ).result;
+ assert.equal(result.metadata.sastbench.status, "ok");
+ assert.equal(result.namedScores.strict_tp, 1);
+ assert.equal(result.namedScores.strict_fn, 0);
+}
assert.throws(
() =>
parseCaseOutcome(
diff --git a/evals/triage-finding/scripts/test-triage-io.mts b/evals/triage-finding/scripts/test-triage-io.mts
index 9af46dc92..14be2c5f7 100644
--- a/evals/triage-finding/scripts/test-triage-io.mts
+++ b/evals/triage-finding/scripts/test-triage-io.mts
@@ -83,7 +83,7 @@ assertPasses(
outputFor({
inputId: "input-001",
verdict: "confirmed",
- }),
+ }).replace('"evidence": []', '"evidence": ["Inline ``` fence in source."]'),
{
vars: {
case_id: "case-001",