OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(evals): drop redundant triage instruction wording checks (#2074)
Michael D'Angelo9fe805efa928
diff --git a/evals/triage-finding/scripts/test-app-surface-instructions.mts b/evals/triage-finding/scripts/test-app-surface-instructions.mtsindex 91d4e0969..f8a857e23 100644--- a/evals/triage-finding/scripts/test-app-surface-instructions.mts+++ b/evals/triage-finding/scripts/test-app-surface-instructions.mts@@ -4,40 +4,45 @@ import path from "node:path";const pluginRoot = path.resolve(import.meta.dirname,- "..",- "..",- "..",- "plugins",- "codex-security",+ "../../../plugins/codex-security",);-const skillRoot = path.join(pluginRoot, "skills", "triage-finding");-const skillPath = path.join(skillRoot, "SKILL.md");-const skill = fs.readFileSync(skillPath, "utf8");-const ticketIntakePath = path.join(skillRoot, "references", "ticket-intake.md");-const ticketIntake = fs.readFileSync(ticketIntakePath, "utf8");-const agentPath = path.join(skillRoot, "agents", "openai.yaml");-const agent = fs.readFileSync(agentPath, "utf8");-const pluginPath = path.join(pluginRoot, ".codex-plugin", "plugin.json");-const plugin = JSON.parse(fs.readFileSync(pluginPath, "utf8"));+function readPluginFile(relativePath: string) {+ return fs.readFileSync(path.join(pluginRoot, relativePath), "utf8");+}+const skillRoot = "skills/triage-finding";+const skill = readPluginFile(`${skillRoot}/SKILL.md`);+const ticketIntake = readPluginFile(`${skillRoot}/references/ticket-intake.md`);+const agent = readPluginFile(`${skillRoot}/agents/openai.yaml`);+const plugin = JSON.parse(readPluginFile(".codex-plugin/plugin.json"));assert.match(skill, /references\/ticket-intake\.md/);-assert.match(ticketIntake, /Atlassian[\s\S]*JQL/);assert.match(ticketIntake,/\bsearchJiraIssuesUsingJql`?\s+for\s+collections\b|\bfor\s+collections,\s+use\s+`?searchJiraIssuesUsingJql\b/i,);assert.match(ticketIntake, /natural-language search[\s\S]*discover[\s\S]*JQL/);-assert.match(skill, /security or vulnerability Jira\/Linear tickets/);-assert.match(skill, /Atlassian and Linear mentions are connector hints/);-assert.match(skill, /generic ticket or duplicate triage/);-assert.match(- ticketIntake,- /Normalize Jira and Linear vulnerability tickets as `source_type: "scanner_ticket"`/,-);-assert.match(- ticketIntake,- /issue key[\s\S]*URL[\s\S]*project[\s\S]*status[\s\S]*labels[\s\S]*components[\s\S]*priority/,-);+assert.match(ticketIntake, /exhaust[\s\S]*pag(?:es|ination)/i);+assert.match(ticketIntake, /missing connector|connector.*unavailable/i);+assert.match(ticketIntake, /authentication|reauthorize/i);+assert.match(ticketIntake, /insufficient permission|request access/i);+assert.match(ticketIntake, /not found|inaccessible/i);+assert.match(ticketIntake, /transient/i);+const provenance =+ ticketIntake+ .split(/\n\s*\n/u)+ .find((paragraph) => paragraph.includes("normalized_input.references")) ??+ "";+for (const field of [+ "issue key",+ "URL",+ "project",+ "status",+ "labels",+ "components",+ "priority",+]) {+ assert.ok(provenance.includes(field), field);+}const sentences = ticketIntake.split(/[.!?]|\n\s*\n|^#{1,6}[ \t]+.*$/m).map((sentence) => sentence.trim().replace(/\s+/g, " "));@@ -58,15 +63,6 @@ assert.ok(),);assert.match(agent, /default_prompt:.*Use \$triage-finding/);-assert.match(ticketIntake, /missing connector|connector.*unavailable/i);-assert.match(ticketIntake, /authentication|reauthorize/i);-assert.match(ticketIntake, /insufficient permission|request access/i);-assert.match(ticketIntake, /not found|inaccessible/i);-assert.match(ticketIntake, /transient/i);-assert.match(ticketIntake, /exhaust[\s\S]*pag(?:es|ination)/i);-assert.match(ticketIntake, /independent vulnerability claim/i);-assert.match(ticketIntake, /ambiguous[\s\S]*ask/i);-assert.match(ticketIntake, /deterministic[\s\S]*tree order/i);assert.ok(plugin.interface.defaultPrompt.length <= 3);assert(plugin.interface.defaultPrompt.every(@@ -74,7 +70,8 @@ assert(),);assert(- plugin.interface.defaultPrompt.includes(- "Triage existing security findings against this repository.",+ plugin.interface.defaultPrompt.some(+ (prompt: string) =>+ /\btriage\b/i.test(prompt) && /\bfindings?\b/i.test(prompt),),);