Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(evals): choose GitHub REST when no connector is requested (#1051)

Michael D'Angelocfc8eec38e54
diff --git a/evals/triage-finding/README.md b/evals/triage-finding/README.md
index f057a72f4..e6402f6a2 100644
--- a/evals/triage-finding/README.md
+++ b/evals/triage-finding/README.md
@@ -66,7 +66,9 @@ The eval target is `fixtures/repo`, a small synthetic Express app with both true
The explicit Connector case returns a three-field JSON decision for transport,
fallback, and account/repository scope. The code-scanning case returns request
-paths and query parameters as JSON. Other intake cases use freeform answers.
+paths and query parameters as JSON. A separate default-transport case checks
+that intake selects REST without an explicit Connector request. Other intake
+cases use freeform answers.
## Calibration Dataset
diff --git a/evals/triage-finding/assertions/github-rest-intake.mts b/evals/triage-finding/assertions/github-rest-intake.mts
index e57e9277f..fc4fbe41e 100644
--- a/evals/triage-finding/assertions/github-rest-intake.mts
+++ b/evals/triage-finding/assertions/github-rest-intake.mts
@@ -223,6 +223,13 @@ const checks: Record<
return failures;
},
+ default_rest: (text) => {
+ const decision = structuredAnswer(text);
+ return decision?.transport === "rest" && Object.keys(decision).length === 1
+ ? []
+ : ["must select REST when the user has not requested the Connector"];
+ },
+
explicit_issue: (text) => {
return [
...(!/GitHub Issues?.*(explicit|specific)|specific.*GitHub Issues?/is.test(
diff --git a/evals/triage-finding/scripts/test-intake-assertions.mts b/evals/triage-finding/scripts/test-intake-assertions.mts
index 078ecd58b..7895fb554 100644
--- a/evals/triage-finding/scripts/test-intake-assertions.mts
+++ b/evals/triage-finding/scripts/test-intake-assertions.mts
@@ -374,6 +374,48 @@ for (const text of [
]) {
expectPass(github, text, githubContext("explicit_connector"), false);
}
+const restJson = JSON.stringify({ transport: "rest" });
+for (const text of [restJson, `\`\`\`json\n${restJson}\n\`\`\``]) {
+ expectPass(github, text, githubContext("default_rest"));
+}
+for (const invalid of [
+ null,
+ [],
+ {},
+ decision,
+ { transport: "other" },
+ { transport: "rest", findings: [] },
+]) {
+ expectPass(
+ github,
+ JSON.stringify(invalid),
+ githubContext("default_rest"),
+ false,
+ );
+}
+for (const [behavior, answer] of [
+ ["explicit_connector", connectorJson],
+ ["default_rest", restJson],
+]) {
+ for (const extra of [
+ "null",
+ "true",
+ "42",
+ '"extra"',
+ "{}",
+ triage,
+ "{'verdict': 'confirmed'}",
+ '{schema_version: "triage-finding/v0", findings: []}',
+ ]) {
+ for (const text of [
+ `${answer}\n${extra}`,
+ `\`\`\`json\n${answer}\n\`\`\`\n\`\`\`json\n${extra}\n\`\`\``,
+ ]) {
+ expectPass(github, text, githubContext(behavior), false);
+ }
+ }
+}
+
for (const behavior of ["", "code_scaning", "constructor"]) {
expectPass(github, "arbitrary text", githubContext(behavior), false);
}
diff --git a/evals/triage-finding/tests/github-rest-intake.yaml b/evals/triage-finding/tests/github-rest-intake.yaml
index bef43f9e3..27b9c7996 100644
--- a/evals/triage-finding/tests/github-rest-intake.yaml
+++ b/evals/triage-finding/tests/github-rest-intake.yaml
@@ -107,6 +107,27 @@
- type: javascript
value: file://./assertions/github-rest-intake.mts
+- description: "GitHub intake selects REST without an explicit Connector request"
+ metadata:
+ case_id: github-default-rest
+ source_type: github_repo
+ vars:
+ case_id: github-request-7
+ target_repo: https://github.com/promptfoo/promptfoo
+ source_type_under_test: github_repo
+ finding_input: "Source requested: code scanning."
+ invocation_preamble: "Use $codex-security:triage-finding for code scanning in GitHub repo promptfoo/promptfoo."
+ expected_github_rest_behavior: default_rest
+ eval_instructions: |-
+ This is an automated eval of the transport choice. Do not query live
+ GitHub, inspect code, or emit a triage result. Return one JSON object with
+ only transport, chosen from "rest", "github_connector_read_only", or "other".
+ options:
+ disableDefaultAsserts: true
+ assert:
+ - type: javascript
+ value: file://./assertions/github-rest-intake.mts
+
- description: "Explicit GitHub Connector request preserves the selected transport"
metadata:
case_id: github-explicit-connector