Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(evals): parse fenced triage results in one pass (#2214)

Michael D'Angelo627eef708f44
diff --git a/evals/triage-finding/assertions/output.mts b/evals/triage-finding/assertions/output.mts
index c660037eb..7cc844710 100644
--- a/evals/triage-finding/assertions/output.mts
+++ b/evals/triage-finding/assertions/output.mts
@@ -74,36 +74,31 @@ export function parseExpected(
return trim ? terms.flatMap((term) => term.trim() || []) : terms;
}
+function parseTriageJson(candidate: string) {
+ try {
+ const result = JSON.parse(candidate) as TriageResult;
+ if (result?.schema_version === "triage-finding/v0") return result;
+ } catch {
+ // The output may contain other text or invalid fenced blocks.
+ }
+}
+
export function extractTriageResult(
output: unknown,
failureMessage = "Could not find a parseable triage-finding/v0 JSON result",
) {
const text = outputText(output);
- try {
- const parsed = JSON.parse(text) as TriageResult;
- if (parsed && parsed.schema_version === "triage-finding/v0") return parsed;
- } catch {
- // The result may be inside a multiline Markdown fence.
- }
- const candidates: string[] = [];
+ const plain = parseTriageJson(text);
+ if (plain) return plain;
let fenceStart = -1;
for (const match of text.matchAll(/^[\t ]*```([^\r\n]*)\r?$/gm)) {
if (fenceStart < 0) {
fenceStart = match.index + match[0].length;
} else if (/^[\t ]*$/.test(match[1])) {
- candidates.push(text.slice(fenceStart, match.index));
+ const parsed = parseTriageJson(text.slice(fenceStart, match.index));
+ if (parsed) return parsed;
fenceStart = -1;
}
}
- for (const candidate of candidates) {
- try {
- const parsed = JSON.parse(candidate) as TriageResult;
- if (parsed && parsed.schema_version === "triage-finding/v0") {
- return parsed;
- }
- } catch {
- // The response may contain more than one fenced block. Try the next one.
- }
- }
throw new Error(failureMessage);
}