OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(evals): add SSRF discovery and an allowlisted control (#1917)
Michael D'Angelo0aeb39d501e0
diff --git a/evals/standard-time-budget/README.md b/evals/standard-time-budget/README.mdindex 2a2417af5..e6a2f0646 100644--- a/evals/standard-time-budget/README.md+++ b/evals/standard-time-budget/README.md@@ -45,10 +45,13 @@ child processes before cleaning temporary state.## Fixture and isolation-The synthetic application has tenant authorization, SQL injection, file-traversal, and reflected HTML bugs, each with a safe control. Source and-deployment context go to the model; gold labels and expected locations stay-in the evaluator.+The synthetic application covers tenant authorization, SQL injection, file+traversal, reflected HTML, and server-side request forgery (SSRF). Each case+has a safe control. The SSRF pair checks whether a tenant can choose the+destination of a server-side status request. The billing route accepts only+the two approved URLs. Both routes return the status of the first response,+including redirects. The model receives source and deployment context. The+evaluator holds the gold labels and locations.Each arm runs in an opaque temporary directory outside the artifact tree; pairand variant labels appear only in evaluator reports and archives. After the@@ -98,6 +101,9 @@ cached input is a subset of input tokens, and no dollar cost is inferred.## Interpretation and checks+Use the `fixtures.mts` hash in `experiment.json` to identify the corpus revision.+If two runs have different revisions, compare their results on the shared case IDs.+Two pairs are a smoke comparison, not statistical evidence. Review findingswithout variant labels and compare grounded recall, adjudicated falsepositives, completion, tokens, and elapsed time together. More findings or adiff --git a/evals/standard-time-budget/fixtures.mts b/evals/standard-time-budget/fixtures.mtsindex 590fa8bb0..162f668fe 100644--- a/evals/standard-time-budget/fixtures.mts+++ b/evals/standard-time-budget/fixtures.mts@@ -31,6 +31,11 @@ server files, including configuration, must remain private.Preview endpoints render user-provided titles in HTML for the signed-in browser.The deployment does not add an HTML sanitizer or Content-Security-Policy header.The database and export files are deployment resources, not checked-in fixtures.++Partner status checks use Requests to report an HTTP status to the signed-in+user. Only the catalog and shipping status URLs are approved. Both are managed+by the operator. The portal can also reach internal services that tenants cannot+contact directly.`,"src/app.py": `import osimport sqlite3@@ -39,6 +44,7 @@ from flask import Flask, abort, g, sessionfrom .downloads import downloadsfrom .invoices import invoices+from .partners import partnersfrom .preview import previewfrom .search import search@@ -69,6 +75,7 @@ app.register_blueprint(invoices)app.register_blueprint(search)app.register_blueprint(downloads)app.register_blueprint(preview)+app.register_blueprint(partners)`,"src/__init__.py": "","src/invoices.py": `from flask import Blueprint, abort, g, jsonify@@ -159,6 +166,30 @@ def account_preview():def billing_preview():title = escape(request.args.get("title", ""))return Response(f"<h1>{title}</h1>", content_type="text/html")+`,+ "src/partners.py": `import requests+from flask import Blueprint, abort, request++partners = Blueprint("partners", __name__)+PARTNER_STATUS_URLS = {+ "https://catalog.example.test/status",+ "https://shipping.example.test/status",+}++[email protected]("/account/partner-status")+def account_partner_status():+ response = requests.head(request.args["url"], timeout=3, allow_redirects=False)+ return {"status": response.status_code}++[email protected]("/billing/partner-status")+def billing_partner_status():+ url = request.args["url"]+ if url not in PARTNER_STATUS_URLS:+ abort(400)+ response = requests.head(url, timeout=3, allow_redirects=False)+ return {"status": response.status_code}`,"schema.sql": `CREATE TABLE invoices (id INTEGER PRIMARY KEY,@@ -212,6 +243,12 @@ CREATE TABLE catalog ("return Response",["CWE-79"],),+ sourceCase(+ "partner-status-ssrf",+ "src/partners.py",+ 'requests.head(request.args["url"]',+ ["CWE-918"],+ ),],controls: [sourceCase(@@ -236,6 +273,12 @@ CREATE TABLE catalog (["CWE-79"],1,),+ sourceCase(+ "partner-status-allowlist",+ "src/partners.py",+ "requests.head(url,",+ ["CWE-918"],+ ),],};}diff --git a/evals/standard-time-budget/test_grading.mts b/evals/standard-time-budget/test_grading.mtsindex a98ad1e41..ffd04fce2 100644--- a/evals/standard-time-budget/test_grading.mts+++ b/evals/standard-time-budget/test_grading.mts@@ -3,12 +3,16 @@ import test from "node:test";import { createFixture, type Fixture, type SourceCase } from "./fixtures.mts";import { gradeResult, type SourceLocation } from "./grade.mts";-function findingFor(fixture: Fixture, expected: SourceCase) {+function findingFor(+ fixture: Fixture,+ expected: SourceCase,+ role = "root_control",+) {return {title: expected.id,taxonomy: { cwe: [expected.cwes[0]] },locations: [- { path: expected.path, startLine: expected.line, role: "root_control" },+ { path: expected.path, startLine: expected.line, role },] as SourceLocation[],codeEvidence: [{@@ -30,28 +34,66 @@ function completeResult(fixture: Fixture) {};}-test("authentic source evidence matches all four vulnerability families", () => {+test("authentic source evidence matches every vulnerability family", () => {const fixture = createFixture();const result = completeResult(fixture);result.findings[1].locations[0].role = "sink";const report = gradeResult(result, fixture);assert.equal(report.passed, true);assert.equal(report.recall, 1);- assert.equal(report.expectedCount, 4);- assert.equal(report.foundCount, 4);+ assert.equal(report.expectedCount, fixture.positives.length);+ assert.equal(report.foundCount, fixture.positives.length);assert.deepEqual(report.evidenceErrors, []);});-test("a missed bug reduces recall", () => {+test("missing SSRF reduces recall even when every other case is found", () => {const fixture = createFixture();- const result = completeResult(fixture);- result.findings.pop();- const report = gradeResult(result, fixture);- assert.equal(report.recall, 0.75);+ const findings = completeResult(fixture).findings.filter(+ (finding) => !finding.taxonomy.cwe.includes("CWE-918"),+ );+ const report = gradeResult({ findings }, fixture);+ assert.deepEqual(+ report.cases.filter((entry) => !entry.found),+ [{ id: "partner-status-ssrf", found: false }],+ );+ assert.equal(report.recall, 4 / 5);assert.equal(report.passed, false);assert.deepEqual(report.unmatchedFindings, []);});+test("SSRF earns credit at the user-selected destination and flags the allowlisted destination for review", () => {+ const fixture = createFixture();+ const path = "src/partners.py";+ function findingAt(sink: string) {+ const line =+ fixture.files[path].split("\n").findIndex((row) => row.includes(sink)) ++ 1;+ assert.ok(line > 0, `Missing sink: ${sink}`);+ return findingFor(+ fixture,+ { id: "Outbound status request", path, line, cwes: ["CWE-918"] },+ "sink",+ );+ }+ const positive = gradeResult(+ { findings: [findingAt('requests.head(request.args["url"]')] },+ fixture,+ );+ assert.deepEqual(positive.matchedIds, ["partner-status-ssrf"]);+ assert.equal(positive.safeControlFindings, 0);++ const control = gradeResult(+ { findings: [findingAt("requests.head(url,")] },+ fixture,+ );+ assert.deepEqual(control.matchedIds, []);+ assert.deepEqual(+ control.safeControlDetails.map((entry) => entry.controlIds),+ [["partner-status-allowlist"]],+ );+ assert.equal(control.needsAdjudication, true);+});+test("additional fabricated locations invalidate an otherwise grounded finding", () => {const fixture = createFixture();for (const role of ["root_control", "supporting"]) {@@ -65,7 +107,7 @@ test("additional fabricated locations invalidate an otherwise grounded finding",result.findings[0].locations.push({ ...location, role });const report = gradeResult(result, fixture);assert.equal(report.passed, false);- assert.equal(report.recall, 0.75);+ assert.equal(report.foundCount, fixture.positives.length - 1);assert.equal(report.needsAdjudication, true);assert.deepEqual(report.evidenceErrors, [{@@ -84,7 +126,7 @@ test("duplicate reports cannot increase recall", () => {result.findings.push(structuredClone(result.findings[0]));const report = gradeResult(result, fixture);assert.equal(report.recall, 1);- assert.equal(report.foundCount, 4);+ assert.equal(report.foundCount, fixture.positives.length);assert.equal(report.duplicateFindings, 1);});@@ -116,7 +158,7 @@ test("a different CWE, source line, or non-primary role does not earn a gold matconst result = completeResult(fixture);alter(result.findings[0]);const report = gradeResult(result, fixture);- assert.equal(report.recall, 0.75);+ assert.equal(report.foundCount, fixture.positives.length - 1);assert.equal(report.unmatchedFindings.length, 1);assert.equal(report.needsAdjudication, true);assert.equal(report.safeControlFindings, 0);@@ -153,7 +195,7 @@ test("missing, fabricated, misplaced, or unrelated evidence cannot earn recall",const result = completeResult(fixture);alter(result.findings[0]);const report = gradeResult(result, fixture);- assert.equal(report.recall, 0.75);+ assert.equal(report.foundCount, fixture.positives.length - 1);assert.equal(report.passed, false);assert.ok(report.evidenceErrors.length > 0);}