Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(evals): add SSRF discovery and an allowlisted control (#1917)

Michael D'Angelo0aeb39d501e0
diff --git a/evals/standard-time-budget/README.md b/evals/standard-time-budget/README.md
index 2a2417af5..e6a2f0646 100644
--- a/evals/standard-time-budget/README.md
+++ b/evals/standard-time-budget/README.md
@@ -45,10 +45,13 @@ child processes before cleaning temporary state.
## Fixture and isolation
-The synthetic application has tenant authorization, SQL injection, file
-traversal, and reflected HTML bugs, each with a safe control. Source and
-deployment context go to the model; gold labels and expected locations stay
-in the evaluator.
+The synthetic application covers tenant authorization, SQL injection, file
+traversal, reflected HTML, and server-side request forgery (SSRF). Each case
+has a safe control. The SSRF pair checks whether a tenant can choose the
+destination of a server-side status request. The billing route accepts only
+the two approved URLs. Both routes return the status of the first response,
+including redirects. The model receives source and deployment context. The
+evaluator holds the gold labels and locations.
Each arm runs in an opaque temporary directory outside the artifact tree; pair
and variant labels appear only in evaluator reports and archives. After the
@@ -98,6 +101,9 @@ cached input is a subset of input tokens, and no dollar cost is inferred.
## Interpretation and checks
+Use the `fixtures.mts` hash in `experiment.json` to identify the corpus revision.
+If two runs have different revisions, compare their results on the shared case IDs.
+
Two pairs are a smoke comparison, not statistical evidence. Review findings
without variant labels and compare grounded recall, adjudicated false
positives, completion, tokens, and elapsed time together. More findings or a
diff --git a/evals/standard-time-budget/fixtures.mts b/evals/standard-time-budget/fixtures.mts
index 590fa8bb0..162f668fe 100644
--- a/evals/standard-time-budget/fixtures.mts
+++ b/evals/standard-time-budget/fixtures.mts
@@ -31,6 +31,11 @@ server files, including configuration, must remain private.
Preview endpoints render user-provided titles in HTML for the signed-in browser.
The deployment does not add an HTML sanitizer or Content-Security-Policy header.
The database and export files are deployment resources, not checked-in fixtures.
+
+Partner status checks use Requests to report an HTTP status to the signed-in
+user. Only the catalog and shipping status URLs are approved. Both are managed
+by the operator. The portal can also reach internal services that tenants cannot
+contact directly.
`,
"src/app.py": `import os
import sqlite3
@@ -39,6 +44,7 @@ from flask import Flask, abort, g, session
from .downloads import downloads
from .invoices import invoices
+from .partners import partners
from .preview import preview
from .search import search
@@ -69,6 +75,7 @@ app.register_blueprint(invoices)
app.register_blueprint(search)
app.register_blueprint(downloads)
app.register_blueprint(preview)
+app.register_blueprint(partners)
`,
"src/__init__.py": "",
"src/invoices.py": `from flask import Blueprint, abort, g, jsonify
@@ -159,6 +166,30 @@ def account_preview():
def billing_preview():
title = escape(request.args.get("title", ""))
return Response(f"<h1>{title}</h1>", content_type="text/html")
+`,
+ "src/partners.py": `import requests
+from flask import Blueprint, abort, request
+
+partners = Blueprint("partners", __name__)
+PARTNER_STATUS_URLS = {
+ "https://catalog.example.test/status",
+ "https://shipping.example.test/status",
+}
+
+
[email protected]("/account/partner-status")
+def account_partner_status():
+ response = requests.head(request.args["url"], timeout=3, allow_redirects=False)
+ return {"status": response.status_code}
+
+
[email protected]("/billing/partner-status")
+def billing_partner_status():
+ url = request.args["url"]
+ if url not in PARTNER_STATUS_URLS:
+ abort(400)
+ response = requests.head(url, timeout=3, allow_redirects=False)
+ return {"status": response.status_code}
`,
"schema.sql": `CREATE TABLE invoices (
id INTEGER PRIMARY KEY,
@@ -212,6 +243,12 @@ CREATE TABLE catalog (
"return Response",
["CWE-79"],
),
+ sourceCase(
+ "partner-status-ssrf",
+ "src/partners.py",
+ 'requests.head(request.args["url"]',
+ ["CWE-918"],
+ ),
],
controls: [
sourceCase(
@@ -236,6 +273,12 @@ CREATE TABLE catalog (
["CWE-79"],
1,
),
+ sourceCase(
+ "partner-status-allowlist",
+ "src/partners.py",
+ "requests.head(url,",
+ ["CWE-918"],
+ ),
],
};
}
diff --git a/evals/standard-time-budget/test_grading.mts b/evals/standard-time-budget/test_grading.mts
index a98ad1e41..ffd04fce2 100644
--- a/evals/standard-time-budget/test_grading.mts
+++ b/evals/standard-time-budget/test_grading.mts
@@ -3,12 +3,16 @@ import test from "node:test";
import { createFixture, type Fixture, type SourceCase } from "./fixtures.mts";
import { gradeResult, type SourceLocation } from "./grade.mts";
-function findingFor(fixture: Fixture, expected: SourceCase) {
+function findingFor(
+ fixture: Fixture,
+ expected: SourceCase,
+ role = "root_control",
+) {
return {
title: expected.id,
taxonomy: { cwe: [expected.cwes[0]] },
locations: [
- { path: expected.path, startLine: expected.line, role: "root_control" },
+ { path: expected.path, startLine: expected.line, role },
] as SourceLocation[],
codeEvidence: [
{
@@ -30,28 +34,66 @@ function completeResult(fixture: Fixture) {
};
}
-test("authentic source evidence matches all four vulnerability families", () => {
+test("authentic source evidence matches every vulnerability family", () => {
const fixture = createFixture();
const result = completeResult(fixture);
result.findings[1].locations[0].role = "sink";
const report = gradeResult(result, fixture);
assert.equal(report.passed, true);
assert.equal(report.recall, 1);
- assert.equal(report.expectedCount, 4);
- assert.equal(report.foundCount, 4);
+ assert.equal(report.expectedCount, fixture.positives.length);
+ assert.equal(report.foundCount, fixture.positives.length);
assert.deepEqual(report.evidenceErrors, []);
});
-test("a missed bug reduces recall", () => {
+test("missing SSRF reduces recall even when every other case is found", () => {
const fixture = createFixture();
- const result = completeResult(fixture);
- result.findings.pop();
- const report = gradeResult(result, fixture);
- assert.equal(report.recall, 0.75);
+ const findings = completeResult(fixture).findings.filter(
+ (finding) => !finding.taxonomy.cwe.includes("CWE-918"),
+ );
+ const report = gradeResult({ findings }, fixture);
+ assert.deepEqual(
+ report.cases.filter((entry) => !entry.found),
+ [{ id: "partner-status-ssrf", found: false }],
+ );
+ assert.equal(report.recall, 4 / 5);
assert.equal(report.passed, false);
assert.deepEqual(report.unmatchedFindings, []);
});
+test("SSRF earns credit at the user-selected destination and flags the allowlisted destination for review", () => {
+ const fixture = createFixture();
+ const path = "src/partners.py";
+ function findingAt(sink: string) {
+ const line =
+ fixture.files[path].split("\n").findIndex((row) => row.includes(sink)) +
+ 1;
+ assert.ok(line > 0, `Missing sink: ${sink}`);
+ return findingFor(
+ fixture,
+ { id: "Outbound status request", path, line, cwes: ["CWE-918"] },
+ "sink",
+ );
+ }
+ const positive = gradeResult(
+ { findings: [findingAt('requests.head(request.args["url"]')] },
+ fixture,
+ );
+ assert.deepEqual(positive.matchedIds, ["partner-status-ssrf"]);
+ assert.equal(positive.safeControlFindings, 0);
+
+ const control = gradeResult(
+ { findings: [findingAt("requests.head(url,")] },
+ fixture,
+ );
+ assert.deepEqual(control.matchedIds, []);
+ assert.deepEqual(
+ control.safeControlDetails.map((entry) => entry.controlIds),
+ [["partner-status-allowlist"]],
+ );
+ assert.equal(control.needsAdjudication, true);
+});
+
test("additional fabricated locations invalidate an otherwise grounded finding", () => {
const fixture = createFixture();
for (const role of ["root_control", "supporting"]) {
@@ -65,7 +107,7 @@ test("additional fabricated locations invalidate an otherwise grounded finding",
result.findings[0].locations.push({ ...location, role });
const report = gradeResult(result, fixture);
assert.equal(report.passed, false);
- assert.equal(report.recall, 0.75);
+ assert.equal(report.foundCount, fixture.positives.length - 1);
assert.equal(report.needsAdjudication, true);
assert.deepEqual(report.evidenceErrors, [
{
@@ -84,7 +126,7 @@ test("duplicate reports cannot increase recall", () => {
result.findings.push(structuredClone(result.findings[0]));
const report = gradeResult(result, fixture);
assert.equal(report.recall, 1);
- assert.equal(report.foundCount, 4);
+ assert.equal(report.foundCount, fixture.positives.length);
assert.equal(report.duplicateFindings, 1);
});
@@ -116,7 +158,7 @@ test("a different CWE, source line, or non-primary role does not earn a gold mat
const result = completeResult(fixture);
alter(result.findings[0]);
const report = gradeResult(result, fixture);
- assert.equal(report.recall, 0.75);
+ assert.equal(report.foundCount, fixture.positives.length - 1);
assert.equal(report.unmatchedFindings.length, 1);
assert.equal(report.needsAdjudication, true);
assert.equal(report.safeControlFindings, 0);
@@ -153,7 +195,7 @@ test("missing, fabricated, misplaced, or unrelated evidence cannot earn recall",
const result = completeResult(fixture);
alter(result.findings[0]);
const report = gradeResult(result, fixture);
- assert.equal(report.recall, 0.75);
+ assert.equal(report.foundCount, fixture.positives.length - 1);
assert.equal(report.passed, false);
assert.ok(report.evidenceErrors.length > 0);
}