Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(evals): reject SQL injection without an execution sink (#1053)

mldangelo-oai9c052e6e8ab4
diff --git a/evals/triage-finding/scripts/test-triage-io.mts b/evals/triage-finding/scripts/test-triage-io.mts
index 05954cd81..e4f4f9fe1 100644
--- a/evals/triage-finding/scripts/test-triage-io.mts
+++ b/evals/triage-finding/scripts/test-triage-io.mts
@@ -1,9 +1,20 @@
#!/usr/bin/env node
import assert from "assert";
+import fs from "node:fs";
+import path from "node:path";
+import vm from "node:vm";
import triageIo from "../assertions/triage-io.mts";
import type { TriageFinding } from "../types.ts";
-function outputFor({ inputId, verdict }: { inputId: string; verdict: string }) {
+function outputFor({
+ inputId,
+ verdict,
+ sourceType = "cve",
+}: {
+ inputId: string;
+ verdict: string;
+ sourceType?: string;
+}) {
const stackRank = {
rank_queue: verdict === "not_actionable" ? null : verdict,
rank: verdict === "not_actionable" ? null : 1,
@@ -184,4 +195,88 @@ assert.throws(() => triageIo("no json", { vars: {} }), {
message: "Could not find a parseable triage-finding/v0 JSON block.",
});
+// Exercise the actual fixture: SQL-shaped text is returned as data, not executed.
+const fixturePath = path.resolve(
+ import.meta.dirname,
+ "../fixtures/repo/src/server.js",
+);
+type SearchHandler = (
+ request: { query: { q: string } },
+ response: { json(value: { sql: string }): void },
+) => void;
+const routes = new Map<string, SearchHandler>();
+const app = {
+ get: (route: string, handler: SearchHandler) => routes.set(route, handler),
+};
+vm.runInNewContext(
+ fs.readFileSync(fixturePath, "utf8"),
+ {
+ __dirname: path.dirname(fixturePath),
+ module: { exports: {} },
+ require(name: string) {
+ if (name === "express") return () => app;
+ if (name === "path") return path;
+ throw new Error("Unexpected fixture dependency: " + name);
+ },
+ },
+ { filename: fixturePath },
+);
+let response: { sql: string } | undefined;
+const query = "' OR 1=1 --";
+routes.get("/search")!(
+ { query: { q: query } },
+ {
+ json(value) {
+ response = value;
+ },
+ },
+);
+assert.ok(response);
+assert.deepEqual(Object.keys(response), ["sql"]);
+assert.equal(
+ response.sql,
+ "SELECT id, name FROM products WHERE name LIKE '%" + query + "%'",
+);
+
+const inputCases = fs.readFileSync(
+ path.resolve(import.meta.dirname, "../tests/input-types.yaml"),
+ "utf8",
+);
+const scannerCase = inputCases.match(
+ /case_id: scanner-query[\s\S]*?(?=\n- description:|$)/,
+)![0];
+const scannerVars: Record<string, string> = { case_id: "scanner-query" };
+const scannerContext = { vars: scannerVars };
+for (const field of [
+ "expected_ids",
+ "expected_source_types",
+ "expected_verdicts",
+]) {
+ scannerVars[field] = scannerCase.match(new RegExp(field + ": (.+)"))![1];
+}
+function queryVerdict(inputId: string, verdict: string) {
+ return JSON.parse(
+ outputFor({ inputId, sourceType: "scanner_ticket", verdict }).match(
+ /```json\n([\s\S]*?)\n```/,
+ )![1],
+ );
+}
+const queryOutput = queryVerdict("SCAN-SQL-001", "not_actionable");
+queryOutput.findings.push(
+ queryVerdict("SCAN-SQL-TEST-ONLY", "not_actionable").findings[0],
+);
+queryOutput.findings[1].triage_item_id = "triage-002";
+assertPasses(
+ "a SQL text response is not confirmed SQL injection",
+ JSON.stringify(queryOutput),
+ scannerContext,
+);
+queryOutput.findings[0] = queryVerdict("SCAN-SQL-001", "confirmed").findings[0];
+assertFails(
+ "a missing SQL sink cannot earn a confirmed verdict",
+ JSON.stringify(queryOutput),
+ scannerContext,
+ /expected verdict not_actionable/,
+);
+
console.log("triage-io assertion tests passed");
diff --git a/evals/triage-finding/tests/input-types.yaml b/evals/triage-finding/tests/input-types.yaml
index 1c950751c..bd033be59 100644
--- a/evals/triage-finding/tests/input-types.yaml
+++ b/evals/triage-finding/tests/input-types.yaml
@@ -64,7 +64,7 @@
advisory: Safe download endpoint may allow path traversal.
affected path: src/server.js route /safe-download
-- description: "Scanner ticket input: production unsafe query and test-only false positive"
+- description: "Scanner ticket input: SQL text response and test-only false positives"
metadata:
case_id: scanner-query
source_type: scanner_ticket
@@ -74,7 +74,7 @@
source_type_under_test: scanner_ticket
expected_ids: SCAN-SQL-001,SCAN-SQL-TEST-ONLY
expected_source_types: scanner_ticket,scanner_ticket
- expected_verdicts: confirmed,not_actionable
+ expected_verdicts: not_actionable,not_actionable
finding_input: |-
Source type: scanner_ticket
1. input_id: SCAN-SQL-001