OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(evals): align triage graders with supported behavior (#1245)
mldangelo-oai037437dd0e78
diff --git a/evals/triage-finding/README.md b/evals/triage-finding/README.mdindex bb7b6cc9c..3702f401f 100644--- a/evals/triage-finding/README.md+++ b/evals/triage-finding/README.md@@ -2,7 +2,7 @@This Promptfoo suite verifies that `$codex-security:triage-finding` accepts the current supported input source types and returns the expected `triage-finding/v0`JSON result shape. It also covers bare skill invocation with no supplied finding,-which should prompt the user for a finding in a supported format instead of returning triage JSON. GitHub REST intake cases cover the repository-source control flow, endpoint selection, and connector auth-only rule without querying live GitHub during the eval.+which should prompt the user for a finding in a supported format instead of returning triage JSON. GitHub intake cases cover repository-source selection, REST endpoints, and explicit Connector requests without querying live GitHub during the eval.The suite uses the Promptfoo Codex SDK provider because it only needs final assistant output and deterministic assertions. The eval directory owns a small pinned pnpm environment so new cases can be added and run without a separate scratch setup.@@ -51,7 +51,11 @@ The eval target is `fixtures/repo`, a small synthetic Express app with both true- `assertions/missing-input.mts` checks that bare invocation asks for a finding,names supported input formats, and does not emit triage result JSON.- `tests/github-rest-intake.yaml` opts out of default JSON assertions for GitHub repository-source routing cases.-- `assertions/github-rest-intake.mts` checks GitHub source selection, REST endpoint selection, Codex project repository inference, advisory/private-report handling, connector auth-only behavior, and explicit-only GitHub Issue handling.+- `assertions/github-rest-intake.mts` checks GitHub source selection, REST endpoint selection, Codex project repository inference, advisory/private-report handling, explicit Connector requests and approved REST fallback, and explicit-only GitHub Issue handling.++The explicit Connector case returns a three-field JSON decision for transport,+fallback, and account/repository scope. The code-scanning case returns request+paths and query parameters as JSON. Other intake cases use freeform answers.## Calibration Datasetdiff --git a/evals/triage-finding/assertions/github-rest-intake.mts b/evals/triage-finding/assertions/github-rest-intake.mtsindex 1d7c8ccdd..f2714eef4 100644--- a/evals/triage-finding/assertions/github-rest-intake.mts+++ b/evals/triage-finding/assertions/github-rest-intake.mts@@ -1,19 +1,53 @@import { outputText as textFor, hasTriageJson } from "./output.mts";import type { AssertionContext } from "../types.ts";++function repositoryName(value: string) {+ const path = value.includes("://")+ ? new URL(value).pathname+ : value.replace(/^(?:[^@/]+@)?[^/:]+:/, "");+ return path.replace(/^\/|\/$/g, "").replace(/\.git$/, "");+}+function endpointPattern(path: string, queryParts: string[] = []) {- const patterns = [- escapedLiteralPattern(path),- ...queryParts.map((part) => new RegExp(part, "i")),- ];- return (text: string) => patterns.every((pattern) => pattern.test(text));+ return (text: string, context: AssertionContext) => {+ const repository = repositoryName(context.vars.target_repo as string);+ const paths = [path, path.replace("{owner}/{repo}", repository)];+ return (+ paths.some((candidate) => escapedLiteralPattern(candidate).test(text)) &&+ queryParts.every((part) => escapedLiteralPattern(part).test(text))+ );+ };}function escapedLiteralPattern(value: unknown) {return new RegExp(String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i");}-const checks = {- choose_source: (text: string) => {+function structuredAnswer(text: string) {+ try {+ return JSON.parse(+ text.trim().replace(/^```(?:json)?\s*([\s\S]*?)\s*```$/i, "$1"),+ );+ } catch {+ return null;+ }+}++function normalizesAs(text: string, sourceType: string) {+ const field = `(?:(?:"source_type"|'source_type'|\x60?source_type\x60?)[*_]*\\s*:\\s*|\x60?normalize as\\s+)`;+ const value = `(?:"${sourceType}"|'${sourceType}'|\x60${sourceType}\x60|${sourceType})`;+ const normalization = field + "[*_]*" + value + "\x60?";+ return new RegExp(+ `(?:^|[\\s{,(\\[])[*_]*${normalization}[*_]*(?=$|[\\s\x60,}.;:!?\\)\\]])`,+ "i",+ ).test(text);+}++const checks: Record<+ string,+ (text: string, context: AssertionContext) => string[]+> = {+ choose_source: (text) => {const failures = [];if (!/choose|specify|which|select/i.test(text)) {failures.push("must ask the user to choose or specify a GitHub source");@@ -38,8 +72,8 @@ const checks = {return failures;},- project_repo_inference: (text: string, context: AssertionContext) => {- const failures = checks.choose_source(text);+ project_repo_inference: (text, context) => {+ const failures = checks.choose_source(text, context);const expectedRepo = String(context.vars.expected_inferred_repo || "");if (@@ -66,50 +100,66 @@ const checks = {return failures;},- dependabot_malware: (text: string) => {+ dependabot_malware: (text, context) => {const hasEndpoint = endpointPattern("/repos/{owner}/{repo}/dependabot/alerts",["classification=malware", "state=open", "per_page=100"],- )(text);+ )(text, context);return [...(!hasEndpoint? ["must use Dependabot alerts endpoint with classification=malware, state=open, and per_page=100",]: []),- ...(!/source_type:\s*`?advisory`?|normalize as `?advisory`?/i.test(text)+ ...(!normalizesAs(text, "advisory")? ["must say Dependabot malware normalizes as advisory"]: []),];},- code_scanning: (text: string) => {- const hasAlerts = endpointPattern(- "/repos/{owner}/{repo}/code-scanning/alerts",- ["state=open", "per_page=100"],- )(text);- const hasInstances =- /code-scanning\/alerts\/\{alert_number\}\/instances/i.test(text);+ code_scanning: (text, context) => {+ const answer = structuredAnswer(text);+ const repositories = [+ "{owner}/{repo}",+ repositoryName(context.vars.target_repo as string),+ ];+ const requestMatches = (+ request:+ | { path?: unknown; parameters?: { per_page?: unknown } }+ | null+ | undefined,+ suffix: string,+ ) =>+ request &&+ repositories.some(+ (repository) =>+ request.path === `/repos/${repository}/code-scanning/alerts${suffix}`,+ ) &&+ [100, "100"].some((value) => value === request.parameters?.per_page);return [- ...(!hasAlerts- ? [- "must use code scanning alerts endpoint with state=open and per_page=100",- ]+ ...(!requestMatches(answer?.alerts, "") ||+ answer?.alerts?.parameters?.state !== "open"+ ? ["must describe open code scanning alerts with per_page=100"]: []),- ...(!hasInstances- ? ["must fetch code scanning alert instances per alert"]+ ...(!requestMatches(answer?.instances, "/{alert_number}/instances")+ ? ["must describe per-alert instances with per_page=100"]: []),- ...(!/source_type:\s*`?sarif`?|normalize as `?sarif`?/i.test(text)- ? ["must say code scanning normalizes as sarif"]+ ...(answer?.source_type !== "sarif"+ ? ["must normalize code scanning as sarif"]+ : []),+ ...(Object.keys(answer ?? {}).length !== 3 || hasTriageJson(text)+ ? [+ "must return only alerts, instances, and source_type without triage JSON",+ ]: []),];},- advisories_private_reports: (text: string) => {+ advisories_private_reports: (text, context) => {const hasEndpoint = endpointPattern("/repos/{owner}/{repo}/security-advisories",["per_page=100"],- )(text);+ )(text, context);const hasEachState = ["triage", "draft", "published", "closed"].every((state) => new RegExp(`state=${state}`, "i").test(text),);@@ -132,31 +182,44 @@ const checks = {)? ["must identify state=triage as private vulnerability reports"]: []),- ...(!/source_type:\s*`?advisory`?|normalize as `?advisory`?/i.test(text)+ ...(!normalizesAs(text, "advisory")? ["must say advisories/private reports normalize as advisory"]: []),];},- connector_rest_only: (text: string) => {- return [- ...(!/GitHub Connector.*token|connector.*auth token|token.*GitHub Connector/is.test(- text,- )- ? ["must allow GitHub Connector only as an auth token source"]- : []),- ...(!/REST/i.test(text)- ? ["must state that finding retrieval uses REST"]- : []),- ...(!/do not use.*GitHub Connector.*(fetch|retrieve|data|findings)|GitHub Connector.*not.*(fetch|retrieve|data|findings)/is.test(- text,- )- ? ["must say not to use the GitHub Connector for finding retrieval"]- : []),- ];+ explicit_connector: (text, context) => {+ const repository = repositoryName(context.vars.target_repo as string);+ const decision = structuredAnswer(text);+ if (!decision)+ return ["must return the connector decision as a JSON object"];+ const failures = [];+ if (+ Object.keys(decision ?? {}).length !== 3 ||+ Object.keys(decision?.scope ?? {}).length !== 2+ )+ failures.push(+ "must return only transport, fallback, and scope with only account and repository",+ );+ if (decision?.transport !== "github_connector_read_only")+ failures.push(+ "must retrieve findings through the requested read-only GitHub Connector",+ );+ if (decision?.fallback !== "explain_and_request_rest_approval")+ failures.push(+ "must explain the limitation and request approval before REST fallback",+ );+ if (+ decision?.scope?.account !== "user_specified_or_approved" ||+ decision?.scope?.repository !== repository+ )+ failures.push(+ "must scope the REST fallback to the specified account and exact repository",+ );+ return failures;},- explicit_issue: (text: string) => {+ explicit_issue: (text) => {return [...(!/GitHub Issues?.*(explicit|specific)|specific.*GitHub Issues?/is.test(text,@@ -170,7 +233,7 @@ const checks = {"must say GitHub Issues are not included in all/default source selection",]: []),- ...(!/source_type:\s*`?freeform`?|normalize as `?freeform`?/i.test(text)+ ...(!normalizesAs(text, "freeform")? ["must say explicit GitHub Issues normalize as freeform"]: []),];@@ -180,7 +243,7 @@ const checks = {export default (output: unknown, context: AssertionContext) => {const text = textFor(output);const behavior = String(context.vars.expected_github_rest_behavior || "");- const check = checks[behavior as keyof typeof checks];+ const check = Object.hasOwn(checks, behavior) ? checks[behavior] : undefined;const failures = check? check(text, context): [`unknown expected_github_rest_behavior: ${behavior}`];diff --git a/evals/triage-finding/assertions/missing-input.mts b/evals/triage-finding/assertions/missing-input.mtsindex e45866975..a18b25cb6 100644--- a/evals/triage-finding/assertions/missing-input.mts+++ b/evals/triage-finding/assertions/missing-input.mts@@ -1,4 +1,4 @@-import { outputText } from "./output.mts";+import { outputText, hasTriageJson } from "./output.mts";function matchedFormatNames(text: string) {const supportedFormats: [string, RegExp][] = [@@ -40,25 +40,12 @@ export default (output: unknown) => {);}- if (/```(?:json)?\s*[\s\S]*?```/i.test(text)) {+ if (hasTriageJson(text)) {failures.push(- "response must not return a fenced JSON block when no finding was supplied",+ "response must not emit a triage result JSON object when no finding was supplied",);}- for (const pattern of [- /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i,- /["']findings["']\s*:/i,- /["']verdict["']\s*:/i,- ]) {- if (pattern.test(text)) {- failures.push(- "response must not emit a triage result JSON object when no finding was supplied",- );- break;- }- }-return {pass: failures.length === 0,score: failures.length === 0 ? 1 : 0,diff --git a/evals/triage-finding/assertions/output.mts b/evals/triage-finding/assertions/output.mtsindex 0f2959295..88f9c0a08 100644--- a/evals/triage-finding/assertions/output.mts+++ b/evals/triage-finding/assertions/output.mts@@ -5,12 +5,53 @@ export function outputText(output: unknown) {}export function hasTriageJson(text: string) {- return (- /```(?:json)?\s*[\s\S]*?```/i.test(text) ||- /schema_version\s*["']?\s*:\s*["']?triage-finding\/v0/i.test(text) ||- /["']findings["']\s*:/i.test(text) ||- /["']verdict["']\s*:/i.test(text)- );+ for (const [, key, literal] of text.matchAll(+ /("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|(?<=[{,])\s*(?:schema_version|verdict))\s*:\s*("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|(?:triage-finding\/v0|confirmed|needs_review|not_actionable)(?=\s*[,}]))/g,+ )) {+ try {+ const name = /^["']/.test(key)+ ? JSON.parse(key.replace(/^'|'$/g, '"'))+ : key.trim();+ const value = /^["']/.test(literal)+ ? JSON.parse(literal.replace(/^'|'$/g, '"'))+ : literal;+ if (name === "schema_version" && value === "triage-finding/v0")+ return true;+ if (+ name === "verdict" &&+ ["confirmed", "needs_review", "not_actionable"].includes(value)+ )+ return true;+ } catch {+ // Ignore quoted prose that is not a JSON field.+ }+ }+ let depth = 0;+ let start = 0;+ for (const match of text.matchAll(+ /"(?:\\.|[^"\\])*"|(?<!\w)'(?:\\.|[^'\\])*'|[{}]/g,+ )) {+ if (match[0] === "{") {+ if (depth++ === 0) start = match.index;+ } else if (match[0] === "}" && depth > 0 && --depth === 0) {+ try {+ const result = JSON.parse(text.slice(start, match.index + 1));+ if (+ !("schema_version" in result) &&+ Array.isArray(result.findings) &&+ (result.findings.length === 0 ||+ result.findings.some(+ (finding: { message?: unknown } | null) =>+ typeof finding?.message !== "string",+ ))+ )+ return true;+ } catch {+ // Keep prose and incomplete JSON out of the findings-envelope check.+ }+ }+ }+ return false;}export function parseExpected(diff --git a/evals/triage-finding/assertions/ticket-intake.mts b/evals/triage-finding/assertions/ticket-intake.mtsindex d47bc574f..ba51ab2bb 100644--- a/evals/triage-finding/assertions/ticket-intake.mts+++ b/evals/triage-finding/assertions/ticket-intake.mts@@ -1,5 +1,6 @@import { outputText, hasTriageJson } from "./output.mts";import type { AssertionContext } from "../types.ts";+const expectedPatterns: Record<string, RegExp[]> = {unavailable: [/connector|Linear/i,@@ -27,7 +28,7 @@ const expectedSubissuePatterns: Record<string, RegExp[]> = {direct_confirmation: [/SEC-294/i,/SEC-295/i,- /2\s+(?:direct\s+)?sub-issues|two\s+(?:direct\s+)?sub-issues/i,+ /(?:2|two)\s+(?:direct\s+)?(?:sub-issues|children)/i,/include|import/i,/ask|would you|do you want/i,],@@ -54,10 +55,30 @@ export default (output: unknown, context: AssertionContext) => {const text = outputText(output);const behavior = String(context.vars.expected_ticket_failure || "");const subissueBehavior = String(context.vars.expected_linear_subissues || "");- const patterns = expectedPatterns[behavior] || [];- const subissuePatterns = expectedSubissuePatterns[subissueBehavior] || [];+ const patterns = Object.hasOwn(expectedPatterns, behavior)+ ? expectedPatterns[behavior]+ : [];+ const subissuePatterns = Object.hasOwn(+ expectedSubissuePatterns,+ subissueBehavior,+ )+ ? expectedSubissuePatterns[subissueBehavior]+ : [];const failures = [];+ if (+ (!behavior && !subissueBehavior) ||+ (behavior && !Object.hasOwn(expectedPatterns, behavior))+ ) {+ failures.push(`unknown expected_ticket_failure: ${behavior}`);+ }+ if (+ subissueBehavior &&+ !Object.hasOwn(expectedSubissuePatterns, subissueBehavior)+ ) {+ failures.push(`unknown expected_linear_subissues: ${subissueBehavior}`);+ }+for (const pattern of patterns) {if (!pattern.test(text))failures.push(`missing recovery detail matching ${pattern}`);diff --git a/evals/triage-finding/assertions/triage-io.mts b/evals/triage-finding/assertions/triage-io.mtsindex 41170d237..c0301d9e2 100644--- a/evals/triage-finding/assertions/triage-io.mts+++ b/evals/triage-finding/assertions/triage-io.mts@@ -111,6 +111,11 @@ export default (output: unknown, context: AssertionContext) => {`${label}: actionable or unresolved finding must use a positive integer rank`,);} else {+ if (rankQueue !== finding.verdict) {+ failures.push(+ `${label}: rank_queue must match verdict ${finding.verdict}, got ${rankQueue}`,+ );+ }const ranks = ranksByQueue.get(rankQueue) || [];ranks.push({ label, rank: rank! });ranksByQueue.set(rankQueue, ranks);diff --git a/evals/triage-finding/package.json b/evals/triage-finding/package.jsonindex 75f869167..282bcd540 100644--- a/evals/triage-finding/package.json+++ b/evals/triage-finding/package.json@@ -14,7 +14,7 @@"test:calibration-generate": "node --experimental-strip-types scripts/test-generate-calibration-tests.mts","test:calibration-hydrate": "node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts","test:dataset": "node --experimental-strip-types scripts/test-calibration-dataset.mts",- "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts",+ "test:deterministic": "node --experimental-strip-types scripts/test-app-surface-instructions.mts && node --experimental-strip-types scripts/test-triage-io.mts && node --experimental-strip-types scripts/test-intake-assertions.mts && node --experimental-strip-types scripts/test-calibration-dataset.mts && node --experimental-strip-types scripts/test-generate-calibration-tests.mts && node --experimental-strip-types scripts/test-calibration-evidence.mts && node --experimental-strip-types scripts/test-hydrate-calibration-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts","test:instructions": "node --experimental-strip-types scripts/test-app-surface-instructions.mts","test:sastbench": "node --experimental-strip-types sastbench/scripts/test-sastbench-lib.mts && node --experimental-strip-types sastbench/scripts/test-install-sastbench.mts && node --experimental-strip-types sastbench/scripts/test-hydrate-sastbench-repos.mts && node --experimental-strip-types sastbench/scripts/test-sastbench-result.mts && node --experimental-strip-types sastbench/scripts/test-promptfoo-native-harness.mts","validate": "PROMPTFOO_CONFIG_DIR=.promptfoo PROMPTFOO_DISABLE_WAL_MODE=true promptfoo validate config -c ./promptfooconfig.yaml",diff --git a/evals/triage-finding/scripts/test-intake-assertions.mts b/evals/triage-finding/scripts/test-intake-assertions.mtsnew file mode 100644index 000000000..b8f188fa1--- /dev/null+++ b/evals/triage-finding/scripts/test-intake-assertions.mts@@ -0,0 +1,359 @@+#!/usr/bin/env node++import assert from "node:assert/strict";+import type { AssertionContext } from "../types.ts";+import { hasTriageJson } from "../assertions/output.mts";+import github from "../assertions/github-rest-intake.mts";+import missingInput from "../assertions/missing-input.mts";+import ticket from "../assertions/ticket-intake.mts";++function expectPass(+ grader: typeof github,+ text: string,+ context: AssertionContext = { vars: {} },+ pass = true,+) {+ const result = grader(text, context);+ assert.equal(result.pass, pass, `${text}\n${result.reason}`);+}++const request = "Please provide a SARIF, CVE, advisory, or scanner ticket.";+const triage = JSON.stringify({+ schema_version: "triage-finding/v0",+ findings: [],+});+for (const text of [+ "```sh\ncat finding.sarif\n```",+ '```json\n{"findings": "paste the findings here"}\n```',+ '```json\n{"schema_version": "example/v0", "findings": []}\n```',+ '{"findings":[{"message":"Paste the scanner finding here"}]}',+ '{"input_id":"example","message":"Paste the scanner finding here"}',+ JSON.stringify({ example: '"schema_version": "triage-finding/v0"' }),+ JSON.stringify({ example: '"verdict": "confirmed"' }),+ JSON.stringify({ example: '{"findings":[]}' }),+ JSON.stringify({ example: { findings: [] } }),+ JSON.stringify({ example: { findings: [{}] } }),+ "An example payload is '{\"findings\":[]}'.",+ '{"verdict":"unknown"}',+ JSON.stringify({ example: '{schema_version: "triage-finding/v0"}' }),+ JSON.stringify({ example: "{verdict: confirmed}" }),+ 'The result later uses schema_version: "triage-finding/v0".',+]) {+ assert.equal(hasTriageJson(text), false, text);+ expectPass(missingInput, `${request}\n${text}`);+}+for (const text of [+ ...[[], [{}], [{ input_id: "synthetic", source_type: "freeform" }]].flatMap(+ (findings) => {+ const payload = JSON.stringify({ findings });+ return [+ payload,+ JSON.stringify({ findings }, null, 2),+ `The triage result is: ${payload}`,+ `\`\`\`json\n${payload}\n\`\`\``,+ `Response: {"error":"forbidden"}\n${payload}`,+ ];+ },+ ),+ "Here's the triage result: {\"findings\":[]}. I'll await your input.",+ triage,+ `\`\`\`json\n${triage}\n\`\`\``,+ `\`\`\`sh\ncat finding.sarif\n\`\`\`\n${triage}`,+ `Response: {"error":"forbidden"}\n${triage}`,+ `\`\`\`json\n{"error":"access denied"}\n\`\`\`\n${triage}`,+ `GET /repos/{owner}/{repo}/code-scanning/alerts\n${triage}`,+ `\`\`\`json\n${triage.replace("triage-finding/v0", "triage-finding\\/v0")}\n\`\`\``,+ `Response: {"error":"forbidden"}\n${triage.replace("triage-finding/v0", "triage-finding\\u002fv0")}`,+ '{"findings":[{"input_id":"SEC-293","verdict":"not_actionable"}]}',+ '{"input_id":"SEC-293","verdict":"not_actionable"}',+ '{"triage_item_id":"triage-001","verdict":"needs_review"}',+ '{"verdict":"confirmed"}',+ '{"verdict":"needs_review"}',+ '{"verdict":"not_actionable"}',+ "```json\n{'verdict': 'confirmed'}\n```",+ "{'schema_version': 'triage-finding/v0', 'findings': []}",+ '{schema_version: "triage-finding/v0", findings: []}',+ '{input_id: "synthetic", verdict: "confirmed"}',+ "{verdict: confirmed}",+ "{verdict: needs_review}",+ "{verdict: not_actionable}",+ "{schema_version: triage-finding/v0, findings: []}",+ '{"ver\\u0064ict":"needs\\u005freview"}',+]) {+ assert.equal(hasTriageJson(text), true, text);+ expectPass(missingInput, `${request}\n${text}`, undefined, false);+ expectPass(+ ticket,+ `Request access or paste the content. ${text}`,+ { vars: { expected_ticket_failure: "permission" } },+ false,+ );+}++const childContext = {+ vars: { expected_linear_subissues: "direct_confirmation" },+};+for (const count of [+ "2 direct children",+ "two direct children",+ "2 direct sub-issues",+ "two sub-issues",+]) {+ expectPass(+ ticket,+ `There are ${count}: SEC-294 and SEC-295. Would you like to include them?`,+ childContext,+ );+}+expectPass(+ ticket,+ "There are 2 direct children: SEC-294 and SEC-295. Importing them now.",+ childContext,+ false,+);+for (const vars of [+ {},+ { expected_ticket_failure: "permisson" },+ { expected_ticket_failur: "permission" },+ { expected_linear_subissues: "direct_confirmaton" },+ { expected_ticket_failure: "constructor" },+]) {+ expectPass(ticket, "arbitrary text", { vars }, false);+}+for (const [behavior, text] of [+ [+ "unavailable",+ "Connect or authenticate the unavailable Linear connector, or paste the content.",+ ],+ ["permission", "Request access or paste the content."],+ ["not_found", "The identifier was not found. Check it or paste the content."],+ ["transient", "The retry failed after trying once. Paste the content."],+]) {+ expectPass(ticket, text, { vars: { expected_ticket_failure: behavior } });+}++const githubContext = (behavior: string) => ({+ vars: {+ expected_github_rest_behavior: behavior,+ target_repo: "https://github.com/example/project",+ },+});+for (const repository of ["{owner}/{repo}", "example/project"]) {+ for (const sourceType of [+ "source_type: advisory",+ '`source_type: "advisory"`',+ '"source_type": "advisory"',+ "normalize as `advisory`",+ "(source_type: advisory)",+ "[source_type: advisory]",+ "source_type: advisory\n",+ "source_type: advisory.",+ "source_type: advisory!",+ ]) {+ expectPass(+ github,+ `GET /repos/${repository}/dependabot/alerts?classification=malware&state=open&per_page=100. ${sourceType}`,+ githubContext("dependabot_malware"),+ );+ expectPass(+ github,+ `GET /repos/${repository}/security-advisories?per_page=100, with separate state=triage, state=draft, state=published, state=closed requests. Triage is for private vulnerability reports. ${sourceType}`,+ githubContext("advisories_private_reports"),+ );+ }+}+const codeScanning = {+ alerts: {+ path: "/repos/example/project/code-scanning/alerts",+ parameters: { state: "open", per_page: 100 },+ },+ instances: {+ path: "/repos/example/project/code-scanning/alerts/{alert_number}/instances",+ parameters: { per_page: 100 },+ },+ source_type: "sarif",+};+for (const repository of ["{owner}/{repo}", "example/project"]) {+ const answer = JSON.parse(+ JSON.stringify(codeScanning).replaceAll("example/project", repository),+ );+ expectPass(+ github,+ JSON.stringify(answer).replaceAll(":100", ':"100"'),+ githubContext("code_scanning"),+ );+ for (const text of [+ JSON.stringify(answer),+ `\`\`\`json\n${JSON.stringify(answer)}\n\`\`\``,+ ]) {+ expectPass(github, text, githubContext("code_scanning"));+ }+}+for (const invalid of [+ null,+ [],+ {},+ { ...codeScanning, source_type: "advisory" },+ { ...codeScanning, findings: [] },+ { ...codeScanning, schema_version: "triage-finding/v0", findings: [] },+ { ...codeScanning, findings: [{ verdict: "confirmed" }] },+ { ...codeScanning, instances: { ...codeScanning.instances, parameters: {} } },+ {+ ...codeScanning,+ instances: { ...codeScanning.instances, parameters: { per_page: 1000 } },+ },+ {+ ...codeScanning,+ instances: {+ ...codeScanning.instances,+ path: "/repos/example/other/code-scanning/alerts/{alert_number}/instances",+ },+ },+ {+ ...codeScanning,+ alerts: {+ ...codeScanning.alerts,+ parameters: { state: "closed", per_page: 100 },+ },+ },+ {+ ...codeScanning,+ alerts: { ...codeScanning.alerts, parameters: { state: "open" } },+ },+]) {+ expectPass(+ github,+ JSON.stringify(invalid),+ githubContext("code_scanning"),+ false,+ );+}+expectPass(github, "not JSON", githubContext("code_scanning"), false);++expectPass(+ github,+ 'GitHub Issues require an explicit issue and are not included in all sources. source_type: "freeform"',+ githubContext("explicit_issue"),+);+for (const text of [+ 'GET /repos/example/other/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: "advisory"',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: "sarif"',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open. source_type: "advisory"',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"incorrect_source_type":"advisory"}',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"incorrect source_type":"advisory"}',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory-other"}',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory/other"}',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory other"}',+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type_suffix":"advisory"}',+ "GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. source_type: advisory_other",+]) {+ expectPass(github, text, githubContext("dependabot_malware"), false);+}++for (const emphasis of ["**", "*", "__", "_", "***"]) {+ for (const normalization of [+ `source_type: ${emphasis}advisory${emphasis}`,+ `Normalize as ${emphasis}advisory${emphasis}`,+ `${emphasis}source_type${emphasis}: advisory`,+ ]) {+ expectPass(+ github,+ `GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${normalization}.`,+ githubContext("dependabot_malware"),+ );+ }+ for (const normalization of [+ "source_type: advisory",+ "source_type: `advisory`",+ '"source_type": "advisory"',+ "normalize as advisory",+ "normalize as `advisory`",+ "`source_type: advisory`",+ "`normalize as advisory`",+ "Normalize as advisory.",+ "Normalize as advisory: preserve the identifier",+ "Normalization: source_type: advisory",+ ]) {+ expectPass(+ github,+ `GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${emphasis}${normalization}${emphasis}.`,+ githubContext("dependabot_malware"),+ );+ }+ expectPass(+ github,+ `GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. ${emphasis}source_type: advisory_other${emphasis}`,+ githubContext("dependabot_malware"),+ false,+ );+}++const decision = {+ transport: "github_connector_read_only",+ fallback: "explain_and_request_rest_approval",+ scope: {+ account: "user_specified_or_approved",+ repository: "example/project",+ },+};+const connectorJson = JSON.stringify(decision);+for (const text of [connectorJson, `\`\`\`json\n${connectorJson}\n\`\`\``]) {+ expectPass(github, text, githubContext("explicit_connector"));+}+for (const invalid of [+ null,+ [],+ {},+ ...["transport", "fallback", "scope"].map((key) => ({+ ...decision,+ [key]: undefined,+ })),+ ...["rest", "other", "unknown"].map((transport) => ({+ ...decision,+ transport,+ })),+ ...["automatic_rest", "stop", "unknown"].map((fallback) => ({+ ...decision,+ fallback,+ })),+ { ...decision, scope: { ...decision.scope, account: "any" } },+ { ...decision, scope: { ...decision.scope, repository: "example/other" } },+ { ...decision, explanation: "additional field" },+ { ...decision, scope: { ...decision.scope, extra: true } },+]) {+ expectPass(+ github,+ JSON.stringify(invalid),+ githubContext("explicit_connector"),+ false,+ );+}+for (const text of [+ "not JSON",+ `\`\`\`json\n${connectorJson}\n\`\`\`\n\`\`\`json\n{}\n\`\`\``,+]) {+ expectPass(github, text, githubContext("explicit_connector"), false);+}+for (const behavior of ["", "code_scaning", "constructor"]) {+ expectPass(github, "arbitrary text", githubContext(behavior), false);+}++for (const repository of [+ "example/project",+ "https://github.com/example/project/",+ "https://github.com/example/project.git",+ "[email protected]:example/project.git",+ "ssh://[email protected]/example/project.git",+]) {+ const context = githubContext("explicit_connector");+ context.vars.target_repo = repository;+ expectPass(github, connectorJson, context);+ context.vars.expected_github_rest_behavior = "dependabot_malware";+ expectPass(+ github,+ 'GET /repos/example/project/dependabot/alerts?classification=malware&state=open&per_page=100. {"source_type":"advisory"}',+ context,+ );+}++console.log("intake assertion tests passed");diff --git a/evals/triage-finding/scripts/test-triage-io.mts b/evals/triage-finding/scripts/test-triage-io.mtsindex 87c8d8ba0..05954cd81 100644--- a/evals/triage-finding/scripts/test-triage-io.mts+++ b/evals/triage-finding/scripts/test-triage-io.mts@@ -1,6 +1,7 @@#!/usr/bin/env nodeimport assert from "assert";import triageIo from "../assertions/triage-io.mts";+import type { TriageFinding } from "../types.ts";function outputFor({ inputId, verdict }: { inputId: string; verdict: string }) {const stackRank = {@@ -118,6 +119,67 @@ assertFails(/fixed.*not_actionable|negative/,);+function findingFor(+ inputId: string,+ verdict: string,+ rank: number | null,+ rankQueue: string | null = verdict,+) {+ const finding: TriageFinding = JSON.parse(+ outputFor({ inputId, verdict }).split("```json")[1].split("```")[0],+ ).findings[0];+ finding.exploitability_stack_rank = {+ ...finding.exploitability_stack_rank,+ rank,+ rank_queue: rankQueue,+ };+ return finding;+}++const queueFindings = [+ findingFor("confirmed-1", "confirmed", 1),+ findingFor("review-1", "needs_review", 1),+ findingFor("confirmed-2", "confirmed", 2),+ findingFor("closed", "not_actionable", null, null),+];+const contextFor = (findings: TriageFinding[]) => ({+ vars: {+ expected_ids: findings.map((finding) => finding.input_id),+ expected_source_types: findings.map((finding) => finding.source_type),+ expected_verdicts: findings.map((finding) => finding.verdict),+ },+});+const queueContext = contextFor(queueFindings);+const queueOutput = (findings: TriageFinding[]) =>+ JSON.stringify({ schema_version: "triage-finding/v0", findings });+assertPasses(+ "independent verdict queues",+ queueOutput(queueFindings),+ queueContext,+);+for (const [verdict, wrongQueue] of [+ ["confirmed", "needs_review"],+ ["needs_review", "confirmed"],+ ["confirmed", null],+] as const) {+ const findings = [findingFor("mismatch", verdict, 1, wrongQueue)];+ assertFails(+ "queue matches verdict",+ queueOutput(findings),+ contextFor(findings),+ /rank_queue must match verdict/,+ );+}+for (const rank of [1, 3]) {+ const findings = structuredClone(queueFindings);+ findings[2].exploitability_stack_rank!.rank = rank;+ assertFails(+ "ranks remain unique and contiguous",+ queueOutput(findings),+ queueContext,+ /ranks must be contiguous/,+ );+}assert.throws(() => triageIo("no json", { vars: {} }), {message: "Could not find a parseable triage-finding/v0 JSON block.",});diff --git a/evals/triage-finding/tests/github-rest-intake.yaml b/evals/triage-finding/tests/github-rest-intake.yamlindex 0b0e4bd5a..bef43f9e3 100644--- a/evals/triage-finding/tests/github-rest-intake.yaml+++ b/evals/triage-finding/tests/github-rest-intake.yaml@@ -75,9 +75,11 @@invocation_preamble: "Use $codex-security:triage-finding for code scanning alerts in GitHub repo promptfoo/promptfoo."expected_github_rest_behavior: code_scanningeval_instructions: |-- This is an automated eval. Describe the GitHub REST intake that should be- used for this source. Do not query live GitHub, inspect code, or emit the- triage JSON result.+ This is an automated eval. Return only a JSON object describing the REST+ intake, with keys alerts, instances, and source_type. Each request has a+ path and a parameters object with the required query parameters. Use the+ literal {alert_number} placeholder in the instances path. Do not query live+ GitHub, inspect code, or emit the triage JSON result.options:disableDefaultAsserts: trueassert:@@ -105,21 +107,27 @@- type: javascriptvalue: file://./assertions/github-rest-intake.mts-- description: "GitHub Connector request still uses REST-only finding retrieval"+- description: "Explicit GitHub Connector request preserves the selected transport"metadata:- case_id: github-connector-rest-only+ case_id: github-explicit-connectorsource_type: github_repovars:- case_id: github-connector-rest-only+ case_id: github-explicit-connectortarget_repo: https://github.com/promptfoo/promptfoosource_type_under_test: github_repo- finding_input: "User explicitly requested using the GitHub Connector."- invocation_preamble: "Use $codex-security:triage-finding with the GitHub Connector for GitHub repo promptfoo/promptfoo."- expected_github_rest_behavior: connector_rest_only+ finding_input: "Source requested: code scanning, using the GitHub Connector."+ invocation_preamble: "Use $codex-security:triage-finding with the GitHub Connector for code scanning in GitHub repo promptfoo/promptfoo."+ expected_github_rest_behavior: explicit_connectoreval_instructions: |-- This is an automated eval. Describe how this workflow should handle a- user's explicit request to use the GitHub Connector. Do not query live- GitHub, inspect code, or emit the triage JSON result.+ This is an automated eval. Decide how this workflow should handle the+ user's explicit GitHub Connector request, including an unsupported+ endpoint. Do not query live GitHub, inspect code, or emit a triage result.+ Return only a JSON decision with these three fields:+ - transport: one of "rest", "github_connector_read_only", or "other".+ - fallback: one of "automatic_rest", "stop", or+ "explain_and_request_rest_approval".+ - scope: an object with account ("any" or "user_specified_or_approved")+ and repository (the owner/repo identifier for the proposed REST scope).options:disableDefaultAsserts: trueassert: