Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(evals): drop redundant triage instruction wording checks (#2074)

Michael D'Angelo9fe805efa928
diff --git a/evals/triage-finding/scripts/test-app-surface-instructions.mts b/evals/triage-finding/scripts/test-app-surface-instructions.mts
index 91d4e0969..f8a857e23 100644
--- a/evals/triage-finding/scripts/test-app-surface-instructions.mts
+++ b/evals/triage-finding/scripts/test-app-surface-instructions.mts
@@ -4,40 +4,45 @@ import path from "node:path";
const pluginRoot = path.resolve(
import.meta.dirname,
- "..",
- "..",
- "..",
- "plugins",
- "codex-security",
+ "../../../plugins/codex-security",
);
-const skillRoot = path.join(pluginRoot, "skills", "triage-finding");
-const skillPath = path.join(skillRoot, "SKILL.md");
-const skill = fs.readFileSync(skillPath, "utf8");
-const ticketIntakePath = path.join(skillRoot, "references", "ticket-intake.md");
-const ticketIntake = fs.readFileSync(ticketIntakePath, "utf8");
-const agentPath = path.join(skillRoot, "agents", "openai.yaml");
-const agent = fs.readFileSync(agentPath, "utf8");
-const pluginPath = path.join(pluginRoot, ".codex-plugin", "plugin.json");
-const plugin = JSON.parse(fs.readFileSync(pluginPath, "utf8"));
+function readPluginFile(relativePath: string) {
+ return fs.readFileSync(path.join(pluginRoot, relativePath), "utf8");
+}
+const skillRoot = "skills/triage-finding";
+const skill = readPluginFile(`${skillRoot}/SKILL.md`);
+const ticketIntake = readPluginFile(`${skillRoot}/references/ticket-intake.md`);
+const agent = readPluginFile(`${skillRoot}/agents/openai.yaml`);
+const plugin = JSON.parse(readPluginFile(".codex-plugin/plugin.json"));
assert.match(skill, /references\/ticket-intake\.md/);
-assert.match(ticketIntake, /Atlassian[\s\S]*JQL/);
assert.match(
ticketIntake,
/\bsearchJiraIssuesUsingJql`?\s+for\s+collections\b|\bfor\s+collections,\s+use\s+`?searchJiraIssuesUsingJql\b/i,
);
assert.match(ticketIntake, /natural-language search[\s\S]*discover[\s\S]*JQL/);
-assert.match(skill, /security or vulnerability Jira\/Linear tickets/);
-assert.match(skill, /Atlassian and Linear mentions are connector hints/);
-assert.match(skill, /generic ticket or duplicate triage/);
-assert.match(
- ticketIntake,
- /Normalize Jira and Linear vulnerability tickets as `source_type: "scanner_ticket"`/,
-);
-assert.match(
- ticketIntake,
- /issue key[\s\S]*URL[\s\S]*project[\s\S]*status[\s\S]*labels[\s\S]*components[\s\S]*priority/,
-);
+assert.match(ticketIntake, /exhaust[\s\S]*pag(?:es|ination)/i);
+assert.match(ticketIntake, /missing connector|connector.*unavailable/i);
+assert.match(ticketIntake, /authentication|reauthorize/i);
+assert.match(ticketIntake, /insufficient permission|request access/i);
+assert.match(ticketIntake, /not found|inaccessible/i);
+assert.match(ticketIntake, /transient/i);
+const provenance =
+ ticketIntake
+ .split(/\n\s*\n/u)
+ .find((paragraph) => paragraph.includes("normalized_input.references")) ??
+ "";
+for (const field of [
+ "issue key",
+ "URL",
+ "project",
+ "status",
+ "labels",
+ "components",
+ "priority",
+]) {
+ assert.ok(provenance.includes(field), field);
+}
const sentences = ticketIntake
.split(/[.!?]|\n\s*\n|^#{1,6}[ \t]+.*$/m)
.map((sentence) => sentence.trim().replace(/\s+/g, " "));
@@ -58,15 +63,6 @@ assert.ok(
),
);
assert.match(agent, /default_prompt:.*Use \$triage-finding/);
-assert.match(ticketIntake, /missing connector|connector.*unavailable/i);
-assert.match(ticketIntake, /authentication|reauthorize/i);
-assert.match(ticketIntake, /insufficient permission|request access/i);
-assert.match(ticketIntake, /not found|inaccessible/i);
-assert.match(ticketIntake, /transient/i);
-assert.match(ticketIntake, /exhaust[\s\S]*pag(?:es|ination)/i);
-assert.match(ticketIntake, /independent vulnerability claim/i);
-assert.match(ticketIntake, /ambiguous[\s\S]*ask/i);
-assert.match(ticketIntake, /deterministic[\s\S]*tree order/i);
assert.ok(plugin.interface.defaultPrompt.length <= 3);
assert(
plugin.interface.defaultPrompt.every(
@@ -74,7 +70,8 @@ assert(
),
);
assert(
- plugin.interface.defaultPrompt.includes(
- "Triage existing security findings against this repository.",
+ plugin.interface.defaultPrompt.some(
+ (prompt: string) =>
+ /\btriage\b/i.test(prompt) && /\bfindings?\b/i.test(prompt),
),
);