Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(evals): compare production Standard scan time budgets (#1679)

Michael D'Angelo32b24b5d7cdc
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index 9a47bae56..c37f0da58 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -462,6 +462,14 @@ jobs:
- name: Test secret discovery eval contracts
if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --experimental-strip-types --test evals/secret-discovery/test_*.mts
+ - name: Build host plugin for Standard eval integration
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
+ run: |
+ node plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs --output plugins/codex-security/mcp --native host
+ ln -s ../../plugins/codex-security sdk/typescript/_bundled_plugin
+ - name: Test Standard time-budget eval contracts
+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
+ run: node --experimental-strip-types --test evals/standard-time-budget/test_*.mts
- name: Test workflow routing eval contracts
if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
run: node --experimental-strip-types --test evals/workflow-routing/test_*.mts
diff --git a/evals/README.md b/evals/README.md
index 7a4ad74c2..2e3fa50bb 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -11,10 +11,13 @@ being embedded in its source tree or shipped npm runtime.
- [Secret discovery](secret-discovery/README.md): checks whether the production
core audit finds synthetic credentials in source and keeps them in its final
findings, with deterministic grading and harness checks.
+- [Standard time budget](standard-time-budget/README.md): paired comparison of
+ production Standard scans with and without an advisory time allowance on
+ synthetic vulnerabilities and safe controls, with source-evidence grading.
- [Workflow routing](workflow-routing/README.md): checks whether repository-installed
production skills route natural-language requests to the appropriate workflow,
stopping before workflow execution.
-Model runs are opt-in. CI runs the deterministic triage, secret-discovery, and
-workflow-routing helper checks and the real-IPC reducer regression through the
-normal MCP test suite.
+Model runs are opt-in. CI runs the deterministic triage, secret-discovery,
+time-budget, and workflow-routing helper checks and the real-IPC reducer
+regression through the normal MCP test suite.
diff --git a/evals/secret-discovery/fixtures.mts b/evals/secret-discovery/fixtures.mts
index 5d90cf2ce..4a55a1463 100644
--- a/evals/secret-discovery/fixtures.mts
+++ b/evals/secret-discovery/fixtures.mts
@@ -165,7 +165,7 @@ CONTENT_SHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcd
/** Stage source only. Gold labels and secret-value lookup stay in the harness. */
export async function writeFixture(
repo: string,
- fixture: ReturnType<typeof createFixture>,
+ fixture: Pick<ReturnType<typeof createFixture>, "files">,
) {
for (const [relative, contents] of Object.entries(fixture.files)) {
const destination = join(repo, relative);
diff --git a/evals/secret-discovery/runtime.mts b/evals/secret-discovery/runtime.mts
index f2956e1fa..8cdb8b3b8 100644
--- a/evals/secret-discovery/runtime.mts
+++ b/evals/secret-discovery/runtime.mts
@@ -24,7 +24,7 @@ export { bundledCodexSdkEnvironment } from "../../sdk/typescript/src/codex-sdk-e
/** Keep temporary source and the login link alive until cancelled work stops. */
export async function withEvalState(
createHome: (parent?: string) => Promise<string>,
- ambientHome: string,
+ ambientHome: string | undefined,
run: (state: {
home: string;
root: string;
@@ -47,11 +47,13 @@ export async function withEvalState(
root = await realpath(await mkdtemp(join(tmpdir(), "source-audit-")));
// Share saved login updates without importing the caller's configuration.
let auth;
- try {
- const path = await realpath(join(ambientHome, "auth.json"));
- if ((await stat(path)).isFile()) auth = path;
- } catch (error) {
- if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+ if (ambientHome !== undefined) {
+ try {
+ const path = await realpath(join(ambientHome, "auth.json"));
+ if ((await stat(path)).isFile()) auth = path;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+ }
}
// A hard link must stay on the saved login's filesystem, including symlinks.
const allocatedHome = await createHome(auth ? dirname(auth) : undefined);
diff --git a/evals/standard-time-budget/.gitignore b/evals/standard-time-budget/.gitignore
new file mode 100644
index 000000000..30846059d
--- /dev/null
+++ b/evals/standard-time-budget/.gitignore
@@ -0,0 +1 @@
+/reports/
diff --git a/evals/standard-time-budget/README.md b/evals/standard-time-budget/README.md
new file mode 100644
index 000000000..2a2417af5
--- /dev/null
+++ b/evals/standard-time-budget/README.md
@@ -0,0 +1,118 @@
+# Standard Scan time allowance comparison
+
+Compare production Standard scans with and without `maxTimeMinutes`. Both
+arms call `CodexSecurity.run`, including setup, delegation, canonical artifact
+validation, report generation, and completion. The treatment uses the shared
+[investigation-time-budget.md](../../plugins/codex-security/references/investigation-time-budget.md)
+policy, not an eval-only prompt.
+
+The SDK stamps one shared UTC deadline after setup, immediately before the
+first scan turn. The allowance is advisory: there is no abort timer, automatic
+continuation, or minimum runtime. Finalization is outside the allowance.
+
+## Run
+
+Use the repository's dependency and
+[native-artifact setup](../../plugins/codex-security/native/README.md), then run
+from the repository root:
+
+```sh
+pnpm --dir sdk/typescript run build:plugin
+pnpm --dir sdk/typescript run build:evals
+node --experimental-strip-types evals/standard-time-budget/run.mts
+```
+
+Rebuild the bundle after plugin changes. This developer eval requires an existing
+`OPENAI_API_KEY` or `CODEX_API_KEY` environment variable; saved Codex or
+`codex-security login` credentials are intentionally unsupported. It uses the
+bundled Codex SDK without reading saved logins, provisioning credentials, or
+changing persistent configuration. Production SDK and CLI login support is
+unchanged. **Model runs consume usage and have no enforced time or spend cap.**
+
+[experiment.json](experiment.json) selects an explicit model, two pairs
+(four scans including delegated work), and five advisory minutes. Both arms
+use `xhigh` reasoning and production worker settings. Pairs alternate
+baseline-first and treatment-first order. Override settings with a JSON file:
+
+```sh
+node --experimental-strip-types evals/standard-time-budget/run.mts /path/to/experiment.json
+```
+
+The fields are `model`, `pairs`, and `budgetMinutes`; positive fractional
+minutes are accepted. Production deadline validation runs before either arm
+starts. Ctrl+C saves the interrupted attempt, stops future arms, and waits for
+child processes before cleaning temporary state.
+
+## Fixture and isolation
+
+The synthetic application has tenant authorization, SQL injection, file
+traversal, and reflected HTML bugs, each with a safe control. Source and
+deployment context go to the model; gold labels and expected locations stay
+in the evaluator.
+
+Each arm runs in an opaque temporary directory outside the artifact tree; pair
+and variant labels appear only in evaluator reports and archives. After the
+client closes, its scan output is archived and its state is removed before the
+next arm. If state removal fails, the failed attempt is recorded and the
+comparison stops before another arm starts. Gold labels and prior-arm results
+are not next to the model's working directory or included in prompts. This is layout separation, not a filesystem
+security boundary: production API-key authentication and root-read permissions
+remain unchanged. This eval does not test resumption or external publication.
+
+## Results
+
+Artifacts are saved under ignored `reports/`, updated after every attempt:
+
+- `experiment.json`: settings, evaluator/shared-helper, source, full-plugin,
+ and built-SDK file hashes, plus SDK/Codex package versions and the selected
+ Codex executable paths and SHA-256 hashes, including a `CODEX_CLI_PATH` override.
+ npm launchers also record their native executable. Generated reports
+ are excluded from evaluator hashes.
+- `fixture.json`: synthetic source and held-out gold labels for review.
+- `pair-<number>-<variant>/prompt.txt`: the actual first dispatched scan prompt.
+- Per-arm `scan/`, `result.json`, and `report.json`: canonical scan artifacts,
+ semantic output, grading, usage, elapsed time, and exact failure text.
+- `runs.json` and `summary.json`: all attempts and the paired comparison.
+
+Elapsed time is monotonic and includes SDK setup, investigation, finalization,
+scan archival, cleanup, and grading, but not subsequent evaluator JSON writing.
+A canonical scan that misses a bug still completed successfully. The runner
+exits nonzero for execution failures, not low recall.
+
+The grader counts unique gold cases with matching CWE, a `root_control` or
+`sink` source location, and authentic source evidence. Other or missing roles
+do not earn automatic recall; unmatched findings require adjudication. It
+consumes the production contract's repository-relative POSIX paths. Duplicates
+do not increase recall.
+Safe-control overlaps, unmatched findings, and citation errors are separate
+signals; overlaps and unmatched findings require adjudication, not automatic
+false-positive labels. Free-text location roles can flag benign context
+citations. Preserve raw scores and record adjudication separately: source
+matching is a heuristic, not independent exploit validation.
+
+The summary separates execution success from completed-run quality. Paired
+deltas are treatment minus baseline and include only complete pairs; excluded
+pairs remain explicit. Each attempt's top-level `usage` includes delegated
+workers, even when completion or grading fails. Missing usage stays unknown,
+cached input is a subset of input tokens, and no dollar cost is inferred.
+
+## Interpretation and checks
+
+Two pairs are a smoke comparison, not statistical evidence. Review findings
+without variant labels and compare grounded recall, adjudicated false
+positives, completion, tokens, and elapsed time together. More findings or a
+longer run alone is not an improvement, and these arms do not have matched
+spend. Expand the corpus if both arms saturate the fixture; use more pairs and
+pinned public repository revisions before drawing quality/cost conclusions.
+
+After the builds above, run deterministic checks without model calls:
+
+```sh
+node --experimental-strip-types --test evals/standard-time-budget/test_*.mts evals/secret-discovery/test_*.mts
+```
+
+The integration test uses the real SDK and canonical artifact helpers with
+mocked model/workbench boundaries, verifying isolated state, completion, prompt
+capture, and parent-plus-worker usage. CI reuses its host native build, builds
+the plugin with `build_mcp_app.mjs --native host`, and links it at
+`sdk/typescript/_bundled_plugin`; universal release artifacts are not required.
diff --git a/evals/standard-time-budget/experiment.json b/evals/standard-time-budget/experiment.json
new file mode 100644
index 000000000..dd10dd2a3
--- /dev/null
+++ b/evals/standard-time-budget/experiment.json
@@ -0,0 +1,5 @@
+{
+ "model": "gpt-5.6-sol",
+ "pairs": 2,
+ "budgetMinutes": 5
+}
diff --git a/evals/standard-time-budget/experiment.mts b/evals/standard-time-budget/experiment.mts
new file mode 100644
index 000000000..397fa62a5
--- /dev/null
+++ b/evals/standard-time-budget/experiment.mts
@@ -0,0 +1,171 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { GradeReport, Result } from "./grade.mts";
+import type { PreparedStandardEval } from "./fixtures.mts";
+import type { StandardRunner, StandardRunOptions } from "./production.mts";
+
+export interface ExperimentConfig {
+ model: string;
+ pairs: number;
+ budgetMinutes: number;
+}
+export type Variant = "baseline" | "time-aware";
+/** Retained arm state must not be exposed to a later arm. */
+export class StateCleanupError extends AggregateError {}
+export type RunReport = GradeReport & {
+ scanId?: string;
+ scanDir?: string;
+ reportPath?: string;
+};
+interface RunMetadata {
+ pair: number;
+ variant: Variant;
+ startedAt: string;
+ advisoryBudgetMinutes: number | null;
+ promptSha256: string | null;
+ elapsedMs: number;
+ usage: unknown;
+}
+type RunOutcome =
+ | { status: "completed"; report: RunReport; error?: never }
+ | { status: "failed"; error: string; report?: never };
+export type RunRecord = RunMetadata & RunOutcome;
+export type CompletedRunRecord = Extract<RunRecord, { status: "completed" }>;
+interface ComparisonOptions {
+ prepared: PreparedStandardEval;
+ config: ExperimentConfig;
+ signal?: AbortSignal;
+ run: StandardRunner;
+ elapsedNow?: () => number;
+ onStart?: (record: Pick<RunMetadata, "pair" | "variant">) => Promise<unknown>;
+ onResult?: (
+ record: RunRecord,
+ result: Result | undefined,
+ records: RunRecord[],
+ prompt: string | undefined,
+ ) => Promise<unknown>;
+}
+
+export function validateExperiment(config: ExperimentConfig) {
+ if (typeof config?.model !== "string" || !config.model.trim()) {
+ throw new Error("The experiment requires an explicit model.");
+ }
+ if (!Number.isSafeInteger(config.pairs) || config.pairs < 1) {
+ throw new Error("Experiment pairs must be a positive integer.");
+ }
+ if (!Number.isFinite(config.budgetMinutes) || config.budgetMinutes <= 0) {
+ throw new Error("Experiment budgetMinutes must be a positive number.");
+ }
+ return {
+ model: config.model,
+ pairs: config.pairs,
+ budgetMinutes: config.budgetMinutes,
+ };
+}
+
+export function hashText(text: string | Uint8Array) {
+ return createHash("sha256").update(text).digest("hex");
+}
+
+export async function snapshotEvaluator(directory = import.meta.dirname) {
+ const files: Record<string, string> = {};
+ for (const path of [
+ "run.mts",
+ "production.mts",
+ "experiment.mts",
+ "fixtures.mts",
+ "grade.mts",
+ "summary.mts",
+ "../secret-discovery/fixtures.mts",
+ "../secret-discovery/runtime.mts",
+ ]) {
+ files[path] = hashText(await readFile(join(directory, path)));
+ }
+ return files;
+}
+
+export async function snapshotFiles(directory: string, prefix = "") {
+ const files: Record<string, string> = {};
+ const entries = await readdir(directory, { withFileTypes: true });
+ for (const entry of entries.sort((first, second) =>
+ first.name.localeCompare(second.name),
+ )) {
+ const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
+ const path = join(directory, entry.name);
+ if (entry.isDirectory()) {
+ Object.assign(files, await snapshotFiles(path, relative));
+ } else {
+ files[relative] = hashText(await readFile(path));
+ }
+ }
+ return files;
+}
+
+export async function runComparison({
+ prepared,
+ config,
+ signal,
+ run,
+ elapsedNow = () => performance.now(),
+ onStart = async () => {},
+ onResult = async () => {},
+}: ComparisonOptions) {
+ config = validateExperiment(config);
+ const records: RunRecord[] = [];
+ for (let pair = 1; pair <= config.pairs; pair += 1) {
+ const variants: Variant[] =
+ pair % 2 === 1 ? ["baseline", "time-aware"] : ["time-aware", "baseline"];
+ for (const variant of variants) {
+ signal?.throwIfAborted();
+ await onStart({ pair, variant });
+ signal?.throwIfAborted();
+ const startedAt = new Date().toISOString();
+ const startedElapsed = elapsedNow();
+ const observation: StandardRunOptions["observation"] = {};
+ let outcome: RunOutcome;
+ let semanticResult: Result | undefined;
+ let cleanupFailure: StateCleanupError | undefined;
+ try {
+ const result = await run(prepared, {
+ model: config.model,
+ pair,
+ variant,
+ signal,
+ ...(variant === "time-aware"
+ ? { maxTimeMinutes: config.budgetMinutes }
+ : {}),
+ observation,
+ });
+ outcome = { status: "completed", report: result.report };
+ semanticResult = result.semanticResult;
+ } catch (error) {
+ if (error instanceof StateCleanupError) cleanupFailure = error;
+ outcome = {
+ status: "failed",
+ error: error instanceof Error ? error.message : String(error),
+ };
+ }
+ const elapsedMs = elapsedNow() - startedElapsed;
+ const record: RunRecord = {
+ pair,
+ variant,
+ startedAt,
+ advisoryBudgetMinutes:
+ variant === "time-aware" ? config.budgetMinutes : null,
+ promptSha256:
+ observation.prompt === undefined
+ ? null
+ : hashText(observation.prompt),
+ ...outcome,
+ elapsedMs,
+ usage: observation.usage ?? null,
+ };
+ records.push(record);
+ await onResult(record, semanticResult, records, observation.prompt);
+ if (cleanupFailure !== undefined) throw cleanupFailure;
+ signal?.throwIfAborted();
+ }
+ }
+ return records;
+}
diff --git a/evals/standard-time-budget/fixtures.mts b/evals/standard-time-budget/fixtures.mts
new file mode 100644
index 000000000..590fa8bb0
--- /dev/null
+++ b/evals/standard-time-budget/fixtures.mts
@@ -0,0 +1,254 @@
+export interface SourceCase {
+ id: string;
+ path: string;
+ line: number;
+ cwes: string[];
+}
+
+export interface Fixture {
+ files: Record<string, string>;
+ positives: SourceCase[];
+ controls: SourceCase[];
+}
+
+export function createFixture(): Fixture {
+ const files: Record<string, string> = {
+ "README.md": `# Account portal
+
+This Python Flask service serves authenticated tenant accounts. Both the account
+and billing routes are currently supported. The sign-in service creates signed
+Flask sessions containing a server-selected integer tenant_id; clients cannot
+choose that value. The session signing key comes from deployment configuration.
+
+Invoices and catalog records belong to exactly one tenant. Account users may
+read only their own tenant's records. Invoice identifiers are sequential.
+
+Export downloads are available to all authenticated users, but only files under
+/srv/portal/exports may be downloaded. That directory and its symlinks are
+deployment-managed; account users cannot create or modify files there. Other
+server files, including configuration, must remain private.
+
+Preview endpoints render user-provided titles in HTML for the signed-in browser.
+The deployment does not add an HTML sanitizer or Content-Security-Policy header.
+The database and export files are deployment resources, not checked-in fixtures.
+`,
+ "src/app.py": `import os
+import sqlite3
+
+from flask import Flask, abort, g, session
+
+from .downloads import downloads
+from .invoices import invoices
+from .preview import preview
+from .search import search
+
+app = Flask(__name__)
+app.secret_key = os.environ["SESSION_SIGNING_KEY"]
+app.config.update(SESSION_COOKIE_SECURE=True, SESSION_COOKIE_HTTPONLY=True,
+ SESSION_COOKIE_SAMESITE="Lax")
+
+
+def authenticate():
+ tenant_id = session.get("tenant_id")
+ if type(tenant_id) is not int:
+ abort(401)
+ g.tenant_id = tenant_id
+ g.database = sqlite3.connect(os.environ["PORTAL_DATABASE"])
+ g.database.row_factory = sqlite3.Row
+
+
+def close_database(error):
+ database = g.pop("database", None)
+ if database is not None:
+ database.close()
+
+
+app.register_blueprint(invoices)
+app.register_blueprint(search)
+app.register_blueprint(downloads)
+app.register_blueprint(preview)
+`,
+ "src/__init__.py": "",
+ "src/invoices.py": `from flask import Blueprint, abort, g, jsonify
+
+invoices = Blueprint("invoices", __name__)
+
+
[email protected]("/account/invoices/<int:invoice_id>")
+def account_invoice(invoice_id):
+ row = g.database.execute(
+ "SELECT id, tenant_id, recipient, total FROM invoices WHERE id = ?",
+ (invoice_id,),
+ ).fetchone()
+ if row is None:
+ abort(404)
+ return jsonify(dict(row))
+
+
[email protected]("/billing/invoices/<int:invoice_id>")
+def billing_invoice(invoice_id):
+ row = g.database.execute(
+ "SELECT id, tenant_id, recipient, total FROM invoices WHERE id = ? AND tenant_id = ?",
+ (invoice_id, g.tenant_id),
+ ).fetchone()
+ if row is None:
+ abort(404)
+ return jsonify(dict(row))
+`,
+ "src/search.py": `from flask import Blueprint, g, jsonify, request
+
+search = Blueprint("search", __name__)
+
+
[email protected]("/account/search")
+def account_search():
+ term = request.args.get("q", "")
+ query = f"SELECT id, name FROM catalog WHERE tenant_id = {g.tenant_id} AND name LIKE '%{term}%'"
+ rows = g.database.execute(query).fetchall()
+ return jsonify([dict(row) for row in rows])
+
+
[email protected]("/billing/search")
+def billing_search():
+ term = request.args.get("q", "")
+ rows = g.database.execute(
+ "SELECT id, name FROM catalog WHERE tenant_id = ? AND name LIKE ?",
+ (g.tenant_id, f"%{term}%"),
+ ).fetchall()
+ return jsonify([dict(row) for row in rows])
+`,
+ "src/downloads.py": `from pathlib import Path
+
+from flask import Blueprint, abort, request, send_file
+
+downloads = Blueprint("downloads", __name__)
+EXPORT_ROOT = Path("/srv/portal/exports")
+
+
[email protected]("/account/download")
+def account_download():
+ name = request.args["name"]
+ return send_file(EXPORT_ROOT / name, as_attachment=True)
+
+
[email protected]("/billing/download")
+def billing_download():
+ name = request.args["name"]
+ root = EXPORT_ROOT.resolve()
+ target = (root / name).resolve()
+ if not target.is_relative_to(root):
+ abort(403)
+ return send_file(target, as_attachment=True)
+`,
+ "src/preview.py": `from html import escape
+
+from flask import Blueprint, Response, request
+
+preview = Blueprint("preview", __name__)
+
+
[email protected]("/account/preview")
+def account_preview():
+ title = request.args.get("title", "")
+ return Response(f"<h1>{title}</h1>", content_type="text/html")
+
+
[email protected]("/billing/preview")
+def billing_preview():
+ title = escape(request.args.get("title", ""))
+ return Response(f"<h1>{title}</h1>", content_type="text/html")
+`,
+ "schema.sql": `CREATE TABLE invoices (
+ id INTEGER PRIMARY KEY,
+ tenant_id INTEGER NOT NULL,
+ recipient TEXT NOT NULL,
+ total INTEGER NOT NULL
+);
+
+CREATE TABLE catalog (
+ id INTEGER PRIMARY KEY,
+ tenant_id INTEGER NOT NULL,
+ name TEXT NOT NULL
+);
+`,
+ };
+ const sourceCase = (
+ id: string,
+ path: string,
+ marker: string,
+ cwes: string[],
+ occurrence = 0,
+ ): SourceCase => {
+ const matchingLines = files[path]
+ .split("\n")
+ .flatMap((line, index) => (line.includes(marker) ? [index + 1] : []));
+ const line = matchingLines[occurrence];
+ if (!line) throw new Error(`Fixture marker missing: ${id}`);
+ return { id, path, line, cwes };
+ };
+ return {
+ files,
+ positives: [
+ sourceCase(
+ "invoice-tenant-isolation",
+ "src/invoices.py",
+ 'FROM invoices WHERE id = ?"',
+ ["CWE-639", "CWE-862", "CWE-863", "CWE-284"],
+ ),
+ sourceCase("catalog-sql-injection", "src/search.py", "query = f", [
+ "CWE-89",
+ ]),
+ sourceCase(
+ "export-path-traversal",
+ "src/downloads.py",
+ "send_file(EXPORT_ROOT",
+ ["CWE-22", "CWE-23", "CWE-36", "CWE-73"],
+ ),
+ sourceCase(
+ "preview-reflected-html",
+ "src/preview.py",
+ "return Response",
+ ["CWE-79"],
+ ),
+ ],
+ controls: [
+ sourceCase(
+ "invoice-scoped-query",
+ "src/invoices.py",
+ "AND tenant_id = ?",
+ ["CWE-639", "CWE-862", "CWE-863", "CWE-284"],
+ ),
+ sourceCase("catalog-bound-parameters", "src/search.py", "name LIKE ?", [
+ "CWE-89",
+ ]),
+ sourceCase(
+ "export-contained-path",
+ "src/downloads.py",
+ "send_file(target",
+ ["CWE-22", "CWE-23", "CWE-36", "CWE-73"],
+ ),
+ sourceCase(
+ "preview-escaped-html",
+ "src/preview.py",
+ "return Response",
+ ["CWE-79"],
+ 1,
+ ),
+ ],
+ };
+}
+import { join } from "node:path";
+import { writeFixture } from "../secret-discovery/fixtures.mts";
+
+export async function prepareStandardEval(root: string, runtime: string) {
+ const fixture = createFixture();
+ const repo = join(root, "repository");
+ await writeFixture(repo, fixture);
+ return { repo, runtime, fixture };
+}
+
+export type PreparedStandardEval = Awaited<
+ ReturnType<typeof prepareStandardEval>
+>;
diff --git a/evals/standard-time-budget/grade.mts b/evals/standard-time-budget/grade.mts
new file mode 100644
index 000000000..985782d97
--- /dev/null
+++ b/evals/standard-time-budget/grade.mts
@@ -0,0 +1,205 @@
+import { posix } from "node:path";
+import type {
+ Finding as ScanFinding,
+ FindingCodeEvidence,
+ FindingLocation,
+} from "../../sdk/typescript/src/models.js";
+import type { Fixture, SourceCase } from "./fixtures.mts";
+
+export type SourceLocation = FindingLocation;
+type Finding = Pick<ScanFinding, "title" | "locations"> & {
+ taxonomy: Pick<ScanFinding["taxonomy"], "cwe">;
+ codeEvidence?: Pick<FindingCodeEvidence, "path" | "startLine" | "code">[];
+};
+
+export interface Result {
+ findings: Finding[];
+ [key: string]: unknown;
+}
+
+interface EvidenceError {
+ findingIndex?: number;
+ evidenceIndex?: number;
+ locationIndex?: number;
+ caseId?: string;
+ reason: string;
+}
+
+export type GradeReport = ReturnType<typeof gradeResult>;
+
+function validRange(location: SourceLocation, fixture: Fixture) {
+ const source = fixture.files[posix.normalize(location.path)];
+ const endLine = location.endLine ?? location.startLine;
+ return (
+ typeof source === "string" &&
+ Number.isInteger(location.startLine) &&
+ Number.isInteger(endLine) &&
+ location.startLine > 0 &&
+ endLine >= location.startLine &&
+ endLine <= source.trimEnd().split("\n").length
+ );
+}
+
+function overlaps(location: SourceLocation, expected: SourceCase) {
+ return (
+ posix.normalize(location.path) === expected.path &&
+ location.startLine <= expected.line &&
+ (location.endLine ?? location.startLine) >= expected.line
+ );
+}
+
+function matches(
+ finding: Finding,
+ expected: SourceCase,
+ fixture: Fixture,
+ requirePrimaryRole: boolean,
+) {
+ return (
+ finding.taxonomy.cwe.some((cwe) =>
+ expected.cwes.includes(String(cwe).trim().toUpperCase()),
+ ) &&
+ finding.locations.some(
+ (location) =>
+ (requirePrimaryRole
+ ? /^(root_control|sink)$/u.test(location.role ?? "")
+ : !/^(supporting|support|context|consumer|expected_control)$/u.test(
+ location.role ?? "",
+ )) &&
+ validRange(location, fixture) &&
+ overlaps(location, expected),
+ )
+ );
+}
+
+function inspectEvidence(
+ finding: Finding,
+ fixture: Fixture,
+ findingIndex: number,
+) {
+ const entries = finding.codeEvidence ?? [];
+ const ranges: SourceLocation[] = [];
+ const errors: EvidenceError[] = [];
+ if (!entries.length) {
+ errors.push({ findingIndex, reason: "missing code evidence" });
+ }
+ for (const [evidenceIndex, entry] of entries.entries()) {
+ const code = entry.code.replaceAll("\r\n", "\n").replace(/\n$/u, "");
+ const range = {
+ ...entry,
+ endLine: entry.startLine + code.split("\n").length - 1,
+ };
+ const source = fixture.files[posix.normalize(entry.path)];
+ if (
+ !code.trim() ||
+ !validRange(range, fixture) ||
+ code !==
+ source!
+ .split("\n")
+ .slice(entry.startLine - 1, range.endLine)
+ .join("\n")
+ ) {
+ errors.push({
+ findingIndex,
+ evidenceIndex,
+ reason: "code does not match source at the cited lines",
+ });
+ } else {
+ ranges.push(range);
+ }
+ }
+ return { ranges, errors };
+}
+
+export function gradeResult(result: Result, fixture: Fixture) {
+ const { findings } = result;
+ const evidenceErrors: EvidenceError[] = [];
+ const unmatchedFindings = [];
+ const safeControlDetails = [];
+ const matchCounts = new Map(
+ fixture.positives.map((expected) => [expected.id, 0]),
+ );
+ for (const [findingIndex, finding] of findings.entries()) {
+ const evidence = inspectEvidence(finding, fixture, findingIndex);
+ for (const [locationIndex, location] of finding.locations.entries()) {
+ if (!validRange(location, fixture)) {
+ evidence.errors.push({
+ findingIndex,
+ locationIndex,
+ reason: "location path or line range does not exist in the source",
+ });
+ }
+ }
+ evidenceErrors.push(...evidence.errors);
+ const candidates = fixture.positives.filter((expected) =>
+ matches(finding, expected, fixture, true),
+ );
+ const matched = candidates.filter((expected) => {
+ const grounded = evidence.ranges.some((range) =>
+ overlaps(range, expected),
+ );
+ if (!grounded) {
+ evidenceErrors.push({
+ findingIndex,
+ caseId: expected.id,
+ reason: "code evidence does not cover the expected source line",
+ });
+ }
+ return grounded && evidence.errors.length === 0;
+ });
+ for (const expected of matched) {
+ matchCounts.set(expected.id, matchCounts.get(expected.id)! + 1);
+ }
+ const controls = fixture.controls.filter((expected) =>
+ matches(finding, expected, fixture, false),
+ );
+ if (controls.length) {
+ safeControlDetails.push({
+ findingIndex,
+ title: finding.title,
+ controlIds: controls.map((entry) => entry.id),
+ });
+ }
+ if (!matched.length && !controls.length) {
+ unmatchedFindings.push({
+ findingIndex,
+ title: finding.title,
+ reason: candidates.length
+ ? "expected case lacks authentic evidence"
+ : "no matching source location and CWE in the gold cases",
+ });
+ }
+ }
+ const cases = fixture.positives.map((expected) => ({
+ id: expected.id,
+ found: matchCounts.get(expected.id)! > 0,
+ }));
+ const matchedIds = cases
+ .filter((entry) => entry.found)
+ .map((entry) => entry.id);
+ const foundCount = matchedIds.length;
+ const expectedCount = cases.length;
+ return {
+ passed:
+ foundCount === expectedCount &&
+ safeControlDetails.length === 0 &&
+ unmatchedFindings.length === 0 &&
+ evidenceErrors.length === 0,
+ recall: foundCount / expectedCount,
+ foundCount,
+ expectedCount,
+ matchedIds,
+ cases,
+ duplicateFindings: [...matchCounts.values()].reduce(
+ (total, count) => total + Math.max(0, count - 1),
+ 0,
+ ),
+ safeControlFindings: safeControlDetails.length,
+ safeControlDetails,
+ unmatchedFindings,
+ evidenceErrors,
+ needsAdjudication:
+ unmatchedFindings.length > 0 || safeControlDetails.length > 0,
+ gradingMethod:
+ "Heuristic source-line, CWE, and verbatim evidence matching; not independent exploit validation. Unmatched findings require manual adjudication and are not automatically false positives.",
+ };
+}
diff --git a/evals/standard-time-budget/production.mts b/evals/standard-time-budget/production.mts
new file mode 100644
index 000000000..ab634b358
--- /dev/null
+++ b/evals/standard-time-budget/production.mts
@@ -0,0 +1,183 @@
+import { randomUUID } from "node:crypto";
+import { cp, mkdir, readFile, rm } from "node:fs/promises";
+import { join, relative } from "node:path";
+import { Codex } from "../../sdk/typescript/node_modules/@openai/codex-sdk/dist/index.js";
+import * as sdk from "../../sdk/typescript/dist/api.js";
+import { deepScanOptions } from "../../sdk/typescript/dist/deep-config.js";
+import { createProfileCodex } from "../../sdk/typescript/dist/provider-profile.js";
+import {
+ codexExecutableReadPaths,
+ resolveCodexCommand,
+} from "../../sdk/typescript/dist/runtime.js";
+import type { CodexSecurity as Client } from "../../sdk/typescript/src/api.js";
+import type { PreparedStandardEval } from "./fixtures.mts";
+import { gradeResult, type Result } from "./grade.mts";
+import {
+ hashText,
+ StateCleanupError,
+ type RunReport,
+ type Variant,
+} from "./experiment.mts";
+
+const { CodexSecurity, scanAuthentication } =
+ sdk as unknown as typeof import("../../sdk/typescript/src/api.js");
+type Dependencies = NonNullable<ConstructorParameters<typeof CodexSecurity>[1]>;
+
+export async function snapshotCodexExecutable(environment: NodeJS.ProcessEnv) {
+ const command = resolveCodexCommand(environment).command;
+ const paths = await codexExecutableReadPaths(command);
+ return {
+ command,
+ files: await Promise.all(
+ paths.map(async (path) => ({
+ path,
+ sha256: hashText(await readFile(path)),
+ })),
+ ),
+ };
+}
+
+export function preflightStandardEval(
+ environment: NodeJS.ProcessEnv,
+ budgetMinutes: number,
+) {
+ if (scanAuthentication(environment).method !== "api_key") {
+ throw new Error(
+ "The Standard time-budget eval requires OPENAI_API_KEY or CODEX_API_KEY. Saved logins are not supported by this developer eval.",
+ );
+ }
+ deepScanOptions({ mode: "standard", maxTimeMinutes: budgetMinutes });
+}
+
+export interface StandardRunOptions {
+ model: string;
+ pair: number;
+ variant: Variant;
+ maxTimeMinutes?: number;
+ signal?: AbortSignal;
+ observation: { prompt?: string; usage?: unknown };
+}
+
+export type StandardRunner = (
+ prepared: PreparedStandardEval,
+ options: StandardRunOptions,
+) => Promise<{ report: RunReport; semanticResult: Result }>;
+
+export function createStandardRunner({
+ stateRoot,
+ outputRoot,
+ environment = process.env,
+ createClient = (config, dependencies) =>
+ new CodexSecurity(config, dependencies, { surface: "sdk" }),
+ createCodex = ({ nativeProfile, ...settings }) =>
+ nativeProfile === undefined
+ ? new Codex(settings)
+ : createProfileCodex(settings, nativeProfile),
+}: {
+ stateRoot: string;
+ outputRoot: string;
+ environment?: NodeJS.ProcessEnv;
+ createClient?: (
+ config: ConstructorParameters<typeof CodexSecurity>[0],
+ dependencies: Dependencies,
+ ) => Pick<Client, "run" | "close">;
+ createCodex?: Dependencies["createCodex"];
+}): StandardRunner {
+ return async (prepared, options) => {
+ const stateDirectory = join(stateRoot, randomUUID());
+ const scanDirectory = join(stateDirectory, "scan");
+ const archiveDirectory = join(
+ outputRoot,
+ `pair-${options.pair}-${options.variant}`,
+ "scan",
+ );
+ await mkdir(scanDirectory, { recursive: true, mode: 0o700 });
+ const client = createClient(
+ {
+ pluginPath: prepared.runtime,
+ codexOverrides: {
+ model: options.model,
+ model_reasoning_effort: "xhigh",
+ },
+ },
+ {
+ environment: {
+ ...environment,
+ CODEX_SECURITY_STATE_DIR: stateDirectory,
+ },
+ createCodex: async (settings) => {
+ const codex = await createCodex(settings);
+ return {
+ startThread: (threadOptions) => {
+ const thread = codex.startThread(threadOptions);
+ return {
+ get id() {
+ return thread.id;
+ },
+ runStreamed: (prompt, turnOptions) => {
+ options.observation.prompt ??= prompt;
+ return thread.runStreamed(prompt, turnOptions);
+ },
+ };
+ },
+ };
+ },
+ },
+ );
+ let failure: unknown;
+ try {
+ const result = await client.run(prepared.repo, {
+ mode: "standard",
+ maxTimeMinutes: options.maxTimeMinutes,
+ signal: options.signal,
+ onUsage: (usage) => {
+ options.observation.usage = usage;
+ },
+ outputDir: scanDirectory,
+ });
+ options.observation.usage =
+ result.turnResult.usage ?? options.observation.usage;
+ const semanticResult = {
+ findings: result.findings.findings,
+ coverage: result.coverage,
+ threatModel: result.threatModel,
+ };
+ return {
+ semanticResult,
+ report: {
+ ...gradeResult(semanticResult, prepared.fixture),
+ scanId: result.manifest.scan.id,
+ scanDir: archiveDirectory,
+ reportPath: join(
+ archiveDirectory,
+ relative(result.scanDir, result.reportPath),
+ ),
+ },
+ };
+ } catch (error) {
+ failure = error;
+ throw error;
+ } finally {
+ const failures: unknown[] = failure === undefined ? [] : [failure];
+ try {
+ await client.close();
+ } catch (error) {
+ failures.push(error);
+ }
+ try {
+ await cp(scanDirectory, archiveDirectory, { recursive: true });
+ } catch (error) {
+ failures.push(error);
+ }
+ try {
+ await rm(stateDirectory, { recursive: true, force: true });
+ } catch (error) {
+ failures.push(error);
+ throw new StateCleanupError(failures, failures.map(String).join("\n"));
+ }
+ if (failures.length > 1)
+ throw new AggregateError(failures, failures.map(String).join("\n"));
+ if (failures.length === 1 && failure === undefined) throw failures[0];
+ }
+ };
+}
diff --git a/evals/standard-time-budget/run.mts b/evals/standard-time-budget/run.mts
new file mode 100644
index 000000000..64b27a8c9
--- /dev/null
+++ b/evals/standard-time-budget/run.mts
@@ -0,0 +1,112 @@
+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+ bundledPluginRoot,
+ createIsolatedHome,
+} from "../../sdk/typescript/dist/runtime.js";
+import {
+ CODEX_EXECUTABLE_VERSION,
+ CODEX_SDK_VERSION,
+ VERSION,
+} from "../../sdk/typescript/dist/version.js";
+import { withEvalState } from "../secret-discovery/runtime.mts";
+import {
+ createStandardRunner,
+ preflightStandardEval,
+ snapshotCodexExecutable,
+} from "./production.mts";
+import { prepareStandardEval } from "./fixtures.mts";
+import {
+ runComparison,
+ snapshotEvaluator,
+ snapshotFiles,
+ validateExperiment,
+} from "./experiment.mts";
+import { summarizeRuns } from "./summary.mts";
+
+const config = validateExperiment(
+ JSON.parse(
+ await readFile(
+ process.argv[2] ?? new URL("./experiment.json", import.meta.url),
+ "utf8",
+ ),
+ ),
+);
+preflightStandardEval(process.env, config.budgetMinutes);
+const codexExecutable = await snapshotCodexExecutable(process.env);
+const pluginRoot = await bundledPluginRoot();
+const reports = fileURLToPath(new URL("./reports/", import.meta.url));
+await mkdir(reports, { recursive: true });
+const reportDirectory = await mkdtemp(join(reports, "run-"));
+const saveJson = (path: string, value: unknown) =>
+ writeFile(path, JSON.stringify(value, null, 2) + "\n");
+console.log(`Eval artifacts: ${reportDirectory}`);
+await withEvalState(
+ createIsolatedHome,
+ undefined,
+ async ({ root, home, signal }) => {
+ const prepared = await prepareStandardEval(root, pluginRoot);
+ await saveJson(join(reportDirectory, "experiment.json"), {
+ ...config,
+ reasoningEffort: "xhigh",
+ execution: "CodexSecurity.run Standard production pipeline",
+ order: "alternating baseline-first and time-aware-first pairs",
+ evaluator: await snapshotEvaluator(),
+ source: await snapshotFiles(prepared.repo),
+ plugin: await snapshotFiles(pluginRoot),
+ sdk: await snapshotFiles(
+ fileURLToPath(new URL("../../sdk/typescript/dist/", import.meta.url)),
+ ),
+ packageVersions: {
+ "@openai/codex-security": VERSION,
+ "@openai/codex-sdk": CODEX_SDK_VERSION,
+ "@openai/codex": CODEX_EXECUTABLE_VERSION,
+ },
+ codexExecutable,
+ });
+ await saveJson(join(reportDirectory, "fixture.json"), prepared.fixture);
+ const records = await runComparison({
+ prepared,
+ config,
+ signal,
+ run: createStandardRunner({
+ stateRoot: join(root, "states"),
+ outputRoot: reportDirectory,
+ environment: { ...process.env, CODEX_HOME: home },
+ }),
+ onStart: async (record) => {
+ const directory = join(
+ reportDirectory,
+ `pair-${record.pair}-${record.variant}`,
+ );
+ await mkdir(directory);
+ console.log(`Pair ${record.pair}/${config.pairs}: ${record.variant}`);
+ },
+ onResult: async (record, semanticResult, completedRecords, prompt) => {
+ const directory = join(
+ reportDirectory,
+ `pair-${record.pair}-${record.variant}`,
+ );
+ if (prompt !== undefined)
+ await writeFile(join(directory, "prompt.txt"), prompt);
+ if (semanticResult !== undefined) {
+ await saveJson(join(directory, "result.json"), semanticResult);
+ }
+ await saveJson(join(directory, "report.json"), record);
+ await saveJson(join(reportDirectory, "runs.json"), completedRecords);
+ await saveJson(
+ join(reportDirectory, "summary.json"),
+ summarizeRuns(completedRecords),
+ );
+ console.log(
+ `${record.variant}: ${record.status}, ${(record.elapsedMs / 1000).toFixed(1)}s`,
+ );
+ },
+ });
+ console.log(JSON.stringify(summarizeRuns(records), null, 2));
+ process.exitCode = records.some((record) => record.status === "failed")
+ ? 1
+ : 0;
+ },
+);
diff --git a/evals/standard-time-budget/summary.mts b/evals/standard-time-budget/summary.mts
new file mode 100644
index 000000000..45a8c18ab
--- /dev/null
+++ b/evals/standard-time-budget/summary.mts
@@ -0,0 +1,143 @@
+import type {
+ CompletedRunRecord,
+ RunRecord,
+ RunReport,
+ Variant,
+} from "./experiment.mts";
+
+const variants = ["baseline", "time-aware"] as const;
+const usageFields = [
+ "input_tokens",
+ "cached_input_tokens",
+ "output_tokens",
+] as const;
+type UsageField = (typeof usageFields)[number];
+
+function mean(values: number[]) {
+ return values.length
+ ? values.reduce((total, value) => total + value, 0) / values.length
+ : null;
+}
+
+function usageValue(record: RunRecord, field: UsageField) {
+ const usage = record.usage as Partial<Record<UsageField, number>> | undefined;
+ const value = usage?.[field];
+ return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function totalTokens(record: RunRecord) {
+ const input = usageValue(record, "input_tokens");
+ const output = usageValue(record, "output_tokens");
+ return input === null || output === null ? null : input + output;
+}
+
+function sumKnown(values: (number | null)[]) {
+ return values.length &&
+ values.every((value): value is number => value !== null)
+ ? values.reduce((total, value) => total + value, 0)
+ : null;
+}
+
+function summarizeArm(records: RunRecord[]) {
+ const completed = records.filter(
+ (record): record is CompletedRunRecord => record.status === "completed",
+ );
+ const qualityCount = (getValue: (report: RunReport) => number) =>
+ completed.length
+ ? completed.reduce((total, record) => total + getValue(record.report), 0)
+ : null;
+ return {
+ attempted: records.length,
+ completed: completed.length,
+ failed: records.length - completed.length,
+ successRate: records.length ? completed.length / records.length : null,
+ meanElapsedMs: mean(records.map((record) => record.elapsedMs)),
+ meanRecall: mean(completed.map((record) => record.report.recall)),
+ foundCount: qualityCount((report) => report.foundCount),
+ expectedCount: qualityCount((report) => report.expectedCount),
+ safeControlFindings: qualityCount((report) => report.safeControlFindings),
+ unmatchedFindings: qualityCount(
+ (report) => report.unmatchedFindings.length,
+ ),
+ evidenceErrors: qualityCount((report) => report.evidenceErrors.length),
+ usage: {
+ ...(Object.fromEntries(
+ usageFields.map((field) => [
+ field,
+ sumKnown(records.map((record) => usageValue(record, field))),
+ ]),
+ ) as Record<UsageField, number | null>),
+ total_tokens: sumKnown(records.map(totalTokens)),
+ missingUsageRuns: records.filter((record) =>
+ usageFields.some((field) => usageValue(record, field) === null),
+ ).length,
+ },
+ };
+}
+
+export function summarizeRuns(records: RunRecord[]) {
+ const pairs = new Map<number, Partial<Record<Variant, RunRecord>>>();
+ for (const record of records) {
+ const variant = record.variant;
+ const pair = pairs.get(record.pair) ?? {};
+ if (pair[variant]) {
+ throw new Error(
+ `Duplicate run for pair ${record.pair}/${record.variant}`,
+ );
+ }
+ pair[variant] = record;
+ pairs.set(record.pair, pair);
+ }
+ const completedPairs = [...pairs.values()].filter(
+ (pair): pair is Record<Variant, CompletedRunRecord> =>
+ variants.every((variant) => pair[variant]?.status === "completed"),
+ );
+ const meanDelta = (
+ getValue: (record: CompletedRunRecord) => number | null,
+ ) => {
+ const deltas = completedPairs.map((pair) => {
+ const baseline = getValue(pair.baseline);
+ const timeAware = getValue(pair["time-aware"]);
+ return baseline === null || timeAware === null
+ ? null
+ : timeAware - baseline;
+ });
+ return deltas.every((delta): delta is number => delta !== null)
+ ? mean(deltas)
+ : null;
+ };
+ return {
+ arms: Object.fromEntries(
+ variants.map((variant) => [
+ variant,
+ summarizeArm(records.filter((record) => record.variant === variant)),
+ ]),
+ ),
+ paired: {
+ totalPairs: pairs.size,
+ completedPairs: completedPairs.length,
+ excludedPairs: pairs.size - completedPairs.length,
+ meanDelta: {
+ recall: meanDelta((record) => record.report.recall),
+ elapsedMs: meanDelta((record) => record.elapsedMs),
+ foundCount: meanDelta((record) => record.report.foundCount),
+ safeControlFindings: meanDelta(
+ (record) => record.report.safeControlFindings,
+ ),
+ unmatchedFindings: meanDelta(
+ (record) => record.report.unmatchedFindings.length,
+ ),
+ evidenceErrors: meanDelta(
+ (record) => record.report.evidenceErrors.length,
+ ),
+ ...(Object.fromEntries(
+ usageFields.map((field) => [
+ field,
+ meanDelta((record) => usageValue(record, field)),
+ ]),
+ ) as Record<UsageField, number | null>),
+ total_tokens: meanDelta(totalTokens),
+ },
+ },
+ };
+}
diff --git a/evals/standard-time-budget/test_experiment.mts b/evals/standard-time-budget/test_experiment.mts
new file mode 100644
index 000000000..97583bc6d
--- /dev/null
+++ b/evals/standard-time-budget/test_experiment.mts
@@ -0,0 +1,204 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { writeFixture } from "../secret-discovery/fixtures.mts";
+import {
+ createFixture,
+ prepareStandardEval,
+ type PreparedStandardEval,
+} from "./fixtures.mts";
+import { gradeResult, type Result } from "./grade.mts";
+import {
+ hashText,
+ runComparison,
+ snapshotEvaluator,
+ snapshotFiles,
+ validateExperiment,
+ type RunRecord,
+} from "./experiment.mts";
+
+const config = { model: "eval-model", pairs: 2, budgetMinutes: 5 };
+const preparedStub: PreparedStandardEval = {
+ repo: "/synthetic/repo",
+ runtime: "/synthetic/runtime",
+ fixture: createFixture(),
+};
+
+function mockResult() {
+ return {
+ report: {
+ ...gradeResult({ findings: [] }, preparedStub.fixture),
+ },
+ semanticResult: { findings: [] },
+ };
+}
+
+test("evaluator provenance tracks implementation and shared helpers, not reports", async (context) => {
+ const root = await mkdtemp(join(tmpdir(), "time-budget-provenance-"));
+ context.after(() => rm(root, { recursive: true, force: true }));
+ const directory = join(root, "standard-time-budget");
+ const files = Object.fromEntries(
+ [
+ "run.mts",
+ "production.mts",
+ "experiment.mts",
+ "fixtures.mts",
+ "grade.mts",
+ "summary.mts",
+ "../secret-discovery/fixtures.mts",
+ "../secret-discovery/runtime.mts",
+ ].map((path) => [path, path]),
+ );
+ await writeFixture(directory, { files });
+ const before = await snapshotEvaluator(directory);
+ assert.deepEqual(
+ before,
+ Object.fromEntries(
+ Object.entries(files).map(([path, text]) => [path, hashText(text)]),
+ ),
+ );
+ await writeFixture(directory, {
+ files: {
+ "grade.mts": "changed grader",
+ "../secret-discovery/runtime.mts": "changed state helper",
+ "reports/run-123/experiment.json": "generated report",
+ },
+ });
+ assert.deepEqual(await snapshotEvaluator(directory), {
+ ...before,
+ "grade.mts": hashText("changed grader"),
+ "../secret-discovery/runtime.mts": hashText("changed state helper"),
+ });
+});
+
+test("pairs stage source without labels, alternate order, and opt in only for treatment", async (context) => {
+ const root = await mkdtemp(join(tmpdir(), "time-budget-eval-"));
+ context.after(() => rm(root, { recursive: true, force: true }));
+ const prepared = await prepareStandardEval(root, "/synthetic/plugin");
+ const sourceBefore = await snapshotFiles(prepared.repo);
+ assert.deepEqual(
+ Object.keys(sourceBefore).sort(),
+ Object.keys(prepared.fixture.files).sort(),
+ );
+ const requests: (number | undefined)[] = [];
+ const records = await runComparison({
+ prepared,
+ config,
+ run: async (actual, options) => {
+ assert.equal(actual, prepared);
+ requests.push(options.maxTimeMinutes);
+ options.observation.prompt = "Captured production prompt";
+ return mockResult();
+ },
+ });
+ assert.deepEqual(requests, [undefined, 5, 5, undefined]);
+ assert.deepEqual(
+ records.map(({ pair, variant }) => [pair, variant]),
+ [
+ [1, "baseline"],
+ [1, "time-aware"],
+ [2, "time-aware"],
+ [2, "baseline"],
+ ],
+ );
+ assert.equal(records[0].promptSha256, hashText("Captured production prompt"));
+});
+
+test("monotonic elapsed time includes runner work but not artifact callbacks", async () => {
+ let elapsed = 0;
+ let calls = 0;
+ const records = await runComparison({
+ prepared: preparedStub,
+ config: { ...config, pairs: 1 },
+ elapsedNow: () => elapsed,
+ onStart: async () => {
+ elapsed += 60_000;
+ },
+ run: async () => {
+ calls += 1;
+ elapsed += calls === 1 ? 1_000 : 400_000;
+ return mockResult();
+ },
+ onResult: async () => {
+ elapsed += 60_000;
+ },
+ });
+ assert.equal(calls, 2);
+ assert.deepEqual(
+ records.map((record) => record.elapsedMs),
+ [1_000, 400_000],
+ );
+ assert.equal(records[1].status, "completed");
+});
+
+test("retains exact failures and still runs the paired arm", async () => {
+ const saved: {
+ record: RunRecord;
+ result: Result | undefined;
+ count: number;
+ }[] = [];
+ let calls = 0;
+ const records = await runComparison({
+ prepared: preparedStub,
+ config: { ...config, pairs: 1 },
+ run: async () => {
+ calls += 1;
+ if (calls === 1)
+ throw new Error("model stream failed: synthetic diagnostic");
+ return mockResult();
+ },
+ onResult: async (record, result, records) =>
+ saved.push({ record, result, count: records.length }),
+ });
+ assert.equal(records[0].status, "failed");
+ assert.equal(records[0].error, "model stream failed: synthetic diagnostic");
+ assert.equal(records[0].promptSha256, null);
+ assert.equal(records[1].status, "completed");
+ assert.deepEqual(
+ saved.map(({ count }) => count),
+ [1, 2],
+ );
+ assert.equal(saved[0].result, undefined);
+});
+
+test("explicit cancellation saves the interrupted attempt and stops the comparison", async () => {
+ const controller = new AbortController();
+ const saved: RunRecord[] = [];
+ let calls = 0;
+ await assert.rejects(
+ runComparison({
+ prepared: preparedStub,
+ config,
+ signal: controller.signal,
+ run: async (_prepared, { signal }) => {
+ calls += 1;
+ controller.abort(new Error("user canceled"));
+ signal!.throwIfAborted();
+ throw new Error("Cancellation should abort the runner");
+ },
+ onResult: async (record) => saved.push(record),
+ }),
+ /user canceled/,
+ );
+ assert.equal(calls, 1);
+ assert.equal(saved.length, 1);
+ assert.equal(saved[0].status, "failed");
+});
+
+test("requires an explicit model and valid experiment sizes without imposing a cap", () => {
+ for (const change of [
+ { model: "" },
+ { pairs: 0 },
+ { pairs: 1.5 },
+ { budgetMinutes: 0 },
+ { budgetMinutes: Infinity },
+ ]) {
+ assert.throws(() => validateExperiment({ ...config, ...change }));
+ }
+ assert.deepEqual(
+ validateExperiment({ ...config, pairs: 100, budgetMinutes: 0.25 }),
+ { ...config, pairs: 100, budgetMinutes: 0.25 },
+ );
+});
diff --git a/evals/standard-time-budget/test_grading.mts b/evals/standard-time-budget/test_grading.mts
new file mode 100644
index 000000000..a98ad1e41
--- /dev/null
+++ b/evals/standard-time-budget/test_grading.mts
@@ -0,0 +1,230 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { createFixture, type Fixture, type SourceCase } from "./fixtures.mts";
+import { gradeResult, type SourceLocation } from "./grade.mts";
+
+function findingFor(fixture: Fixture, expected: SourceCase) {
+ return {
+ title: expected.id,
+ taxonomy: { cwe: [expected.cwes[0]] },
+ locations: [
+ { path: expected.path, startLine: expected.line, role: "root_control" },
+ ] as SourceLocation[],
+ codeEvidence: [
+ {
+ path: expected.path,
+ startLine: expected.line,
+ code: fixture.files[expected.path].split("\n")[expected.line - 1],
+ },
+ ],
+ };
+}
+
+type FixtureFinding = ReturnType<typeof findingFor>;
+
+function completeResult(fixture: Fixture) {
+ return {
+ findings: fixture.positives.map((expected) =>
+ findingFor(fixture, expected),
+ ),
+ };
+}
+
+test("authentic source evidence matches all four vulnerability families", () => {
+ const fixture = createFixture();
+ const result = completeResult(fixture);
+ result.findings[1].locations[0].role = "sink";
+ const report = gradeResult(result, fixture);
+ assert.equal(report.passed, true);
+ assert.equal(report.recall, 1);
+ assert.equal(report.expectedCount, 4);
+ assert.equal(report.foundCount, 4);
+ assert.deepEqual(report.evidenceErrors, []);
+});
+
+test("a missed bug reduces recall", () => {
+ const fixture = createFixture();
+ const result = completeResult(fixture);
+ result.findings.pop();
+ const report = gradeResult(result, fixture);
+ assert.equal(report.recall, 0.75);
+ assert.equal(report.passed, false);
+ assert.deepEqual(report.unmatchedFindings, []);
+});
+
+test("additional fabricated locations invalidate an otherwise grounded finding", () => {
+ const fixture = createFixture();
+ for (const role of ["root_control", "supporting"]) {
+ for (const location of [
+ { path: "src/invented.py", startLine: 500 },
+ { path: "src/invoices.py", startLine: 500 },
+ { path: "src/invoices.py", startLine: 0 },
+ { path: "src/invoices.py", startLine: 8, endLine: 7 },
+ ]) {
+ const result = completeResult(fixture);
+ result.findings[0].locations.push({ ...location, role });
+ const report = gradeResult(result, fixture);
+ assert.equal(report.passed, false);
+ assert.equal(report.recall, 0.75);
+ assert.equal(report.needsAdjudication, true);
+ assert.deepEqual(report.evidenceErrors, [
+ {
+ findingIndex: 0,
+ locationIndex: 1,
+ reason: "location path or line range does not exist in the source",
+ },
+ ]);
+ }
+ }
+});
+
+test("duplicate reports cannot increase recall", () => {
+ const fixture = createFixture();
+ const result = completeResult(fixture);
+ result.findings.push(structuredClone(result.findings[0]));
+ const report = gradeResult(result, fixture);
+ assert.equal(report.recall, 1);
+ assert.equal(report.foundCount, 4);
+ assert.equal(report.duplicateFindings, 1);
+});
+
+test("a different CWE, source line, or non-primary role does not earn a gold match", () => {
+ const fixture = createFixture();
+ const alterations: Array<(finding: FixtureFinding) => void> = [
+ (finding) => {
+ finding.taxonomy.cwe = ["CWE-79"];
+ },
+ (finding) => {
+ finding.locations[0].path = "README.md";
+ },
+ (finding) => {
+ finding.locations[0].startLine = 1;
+ },
+ (finding) => {
+ finding.locations[0].role = "supporting";
+ },
+ (finding) => {
+ finding.locations[0].role = "supporting_context";
+ finding.locations.unshift({
+ path: "README.md",
+ startLine: 1,
+ role: "root_control",
+ });
+ },
+ ];
+ for (const alter of alterations) {
+ const result = completeResult(fixture);
+ alter(result.findings[0]);
+ const report = gradeResult(result, fixture);
+ assert.equal(report.recall, 0.75);
+ assert.equal(report.unmatchedFindings.length, 1);
+ assert.equal(report.needsAdjudication, true);
+ assert.equal(report.safeControlFindings, 0);
+ assert.deepEqual(report.evidenceErrors, []);
+ }
+});
+
+test("missing, fabricated, misplaced, or unrelated evidence cannot earn recall", () => {
+ const fixture = createFixture();
+ const alterations: Array<(finding: FixtureFinding) => void> = [
+ (finding) => {
+ finding.codeEvidence = [];
+ },
+ (finding) => {
+ finding.codeEvidence[0].code = "invented source text";
+ },
+ (finding) => {
+ finding.codeEvidence[0].startLine++;
+ },
+ (finding) => {
+ finding.codeEvidence[0].path = "src/search.py";
+ },
+ (finding) => {
+ finding.codeEvidence = [
+ {
+ path: "src/invoices.py",
+ startLine: 1,
+ code: fixture.files["src/invoices.py"].split("\n")[0],
+ },
+ ];
+ },
+ ];
+ for (const alter of alterations) {
+ const result = completeResult(fixture);
+ alter(result.findings[0]);
+ const report = gradeResult(result, fixture);
+ assert.equal(report.recall, 0.75);
+ assert.equal(report.passed, false);
+ assert.ok(report.evidenceErrors.length > 0);
+ }
+});
+
+test("control accusations and unmatched findings have separate adjudication queues", () => {
+ const fixture = createFixture();
+ const result = completeResult(fixture);
+ result.findings.push(findingFor(fixture, fixture.controls[0]));
+ const unmatched = findingFor(fixture, fixture.positives[1]);
+ unmatched.taxonomy.cwe = ["CWE-400"];
+ result.findings.push(unmatched);
+ const report = gradeResult(result, fixture);
+ assert.equal(report.recall, 1);
+ assert.equal(report.safeControlFindings, 1);
+ assert.deepEqual(report.safeControlDetails[0].controlIds, [
+ fixture.controls[0].id,
+ ]);
+ assert.equal(report.unmatchedFindings.length, 1);
+ assert.equal(report.passed, false);
+ assert.equal(report.needsAdjudication, true);
+});
+
+test("safe-control context does not become a control accusation", () => {
+ const fixture = createFixture();
+ const result = completeResult(fixture);
+ const context = findingFor(fixture, fixture.controls[0]);
+ context.locations[0].role = "expected_control";
+ result.findings[0].locations.push(...context.locations);
+ result.findings[0].codeEvidence.push(...context.codeEvidence);
+ assert.equal(gradeResult(result, fixture).passed, true);
+});
+
+test("descriptive safe-control roles require adjudication rather than a false-positive verdict", () => {
+ const fixture = createFixture();
+ const result = completeResult(fixture);
+ const context = findingFor(fixture, fixture.controls[0]);
+ context.locations[0].role =
+ "safe sibling showing the expected tenant control";
+ result.findings[0].locations.push(...context.locations);
+ result.findings[0].codeEvidence.push(...context.codeEvidence);
+ const report = gradeResult(result, fixture);
+ assert.equal(report.recall, 1);
+ assert.equal(report.safeControlFindings, 1);
+ assert.equal(report.needsAdjudication, true);
+ assert.equal(report.passed, false);
+ assert.deepEqual(report.safeControlDetails[0].controlIds, [
+ fixture.controls[0].id,
+ ]);
+});
+
+test("source snippets may span the sink", () => {
+ const fixture = createFixture();
+ const result = completeResult(fixture);
+ for (const finding of result.findings) {
+ const evidence = finding.codeEvidence[0];
+ const source = fixture.files[evidence.path];
+ evidence.startLine--;
+ evidence.code = source
+ .split("\n")
+ .slice(evidence.startLine - 1, evidence.startLine + 1)
+ .join("\n");
+ }
+ assert.equal(gradeResult(result, fixture).passed, true);
+});
+
+test("fixture source is deterministic and contains no gold identifiers", () => {
+ const fixture = createFixture();
+ assert.deepEqual(createFixture(), fixture);
+ const source = Object.values(fixture.files).join("\n");
+ for (const expected of [...fixture.positives, ...fixture.controls]) {
+ assert.equal(source.includes(expected.id), false);
+ }
+});
diff --git a/evals/standard-time-budget/test_production.mts b/evals/standard-time-budget/test_production.mts
new file mode 100644
index 000000000..22ad5d236
--- /dev/null
+++ b/evals/standard-time-budget/test_production.mts
@@ -0,0 +1,378 @@
+import assert from "node:assert/strict";
+import { existsSync, promises as fs } from "node:fs";
+import { cp, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { syncBuiltinESMExports } from "node:module";
+import { join, relative } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import { temporaryDirectory } from "../../plugins/codex-security/mcp-app/tests/support/temporary-directories.ts";
+import type { ThreadEvent } from "../../sdk/typescript/node_modules/@openai/codex-sdk/dist/index.js";
+import type { JsonObject } from "../../sdk/typescript/src/config.js";
+import * as sdk from "../../sdk/typescript/dist/api.js";
+import { createIsolatedHome } from "../../sdk/typescript/dist/runtime.js";
+import { withEvalState } from "../secret-discovery/runtime.mts";
+import { hashText, runComparison } from "./experiment.mts";
+import {
+ createStandardRunner,
+ preflightStandardEval,
+ snapshotCodexExecutable,
+} from "./production.mts";
+import { prepareStandardEval } from "./fixtures.mts";
+import { summarizeRuns } from "./summary.mts";
+
+const { CodexSecurity } =
+ sdk as unknown as typeof import("../../sdk/typescript/src/api.js");
+
+test("executable provenance follows the configured binary and its actual bytes", async (context) => {
+ const root = await temporaryDirectory("time-budget-executable-");
+ context.after(() => rm(root, { recursive: true, force: true }));
+ const first = join(root, "first.exe");
+ const second = join(root, "second.exe");
+ await writeFile(first, "synthetic executable one");
+ await writeFile(second, "synthetic executable two");
+ const snapshot = await snapshotCodexExecutable({ CODEX_CLI_PATH: first });
+ assert.deepEqual(snapshot, {
+ command: first,
+ files: [
+ {
+ path: await realpath(first),
+ sha256: hashText("synthetic executable one"),
+ },
+ ],
+ });
+ assert.deepEqual(await snapshotCodexExecutable({ CODEX_CLI_PATH: second }), {
+ command: second,
+ files: [
+ {
+ path: await realpath(second),
+ sha256: hashText("synthetic executable two"),
+ },
+ ],
+ });
+ await writeFile(first, "replaced executable");
+ const replaced = await snapshotCodexExecutable({ CODEX_CLI_PATH: first });
+ assert.equal(replaced.command, snapshot.command);
+ assert.equal(replaced.files[0]!.sha256, hashText("replaced executable"));
+ assert.notEqual(replaced.files[0]!.sha256, snapshot.files[0]!.sha256);
+});
+
+test("state cleanup failure saves the attempt and stops before the paired arm", async (context) => {
+ const root = await temporaryDirectory("time-budget-cleanup-");
+ const originalRm = fs.rm;
+ context.after(async () => {
+ context.mock.restoreAll();
+ syncBuiltinESMExports();
+ await originalRm(root, { recursive: true, force: true });
+ });
+ const prepared = await prepareStandardEval(root, "/synthetic/plugin");
+ const outputRoot = join(root, "results");
+ const states: string[] = [];
+ const saved: Awaited<ReturnType<typeof runComparison>> = [];
+ context.mock.method(fs, "rm", async (...args: Parameters<typeof fs.rm>) => {
+ if (states.includes(String(args[0]))) {
+ throw new Error("Synthetic state removal failure");
+ }
+ return originalRm(...args);
+ });
+ syncBuiltinESMExports();
+ await assert.rejects(
+ runComparison({
+ prepared,
+ config: { model: "eval-model", pairs: 2, budgetMinutes: 5 },
+ run: createStandardRunner({
+ stateRoot: join(root, "states"),
+ outputRoot,
+ createClient: (_config, dependencies) => {
+ states.push(dependencies.environment["CODEX_SECURITY_STATE_DIR"]!);
+ return {
+ run: async (_target, options) => {
+ await writeFile(join(options!.outputDir!, "partial.json"), "{}");
+ throw new Error("Synthetic scan failure");
+ },
+ close: async () => {
+ throw new Error("Synthetic client close failure");
+ },
+ };
+ },
+ }),
+ onResult: async (record) => {
+ saved.push(record);
+ },
+ }),
+ /Synthetic state removal failure/,
+ );
+ assert.equal(states.length, 1);
+ assert.equal(saved.length, 1);
+ assert.equal(saved[0]!.status, "failed");
+ assert.match(saved[0]!.error!, /Synthetic scan failure/);
+ assert.match(saved[0]!.error!, /Synthetic client close failure/);
+ assert.match(saved[0]!.error!, /Synthetic state removal failure/);
+ assert.equal(existsSync(states[0]!), true);
+ assert.equal(
+ await readFile(
+ join(outputRoot, "pair-1-baseline", "scan", "partial.json"),
+ "utf8",
+ ),
+ "{}",
+ );
+});
+
+test("the developer eval preflights API-key authentication and the production budget before staging", async () => {
+ assert.throws(
+ () => preflightStandardEval({}, 5),
+ /requires OPENAI_API_KEY or CODEX_API_KEY/,
+ );
+ for (const name of ["OPENAI_API_KEY", "CODEX_API_KEY"]) {
+ preflightStandardEval({ [name]: "synthetic-environment" }, 0.25);
+ }
+ assert.throws(
+ () =>
+ preflightStandardEval({ OPENAI_API_KEY: "synthetic-environment" }, 1e12),
+ /deadline within the supported date range/,
+ );
+ await withEvalState(createIsolatedHome, undefined, async ({ home }) => {
+ assert.equal(existsSync(join(home, "auth.json")), false);
+ });
+});
+
+async function writeUsageSession(
+ home: string,
+ threadId: string,
+ usage: Record<string, number>,
+ parent?: string,
+) {
+ const directory = join(home, "sessions", "2026", "01", "01");
+ await mkdir(directory, { recursive: true });
+ await writeFile(
+ join(directory, `rollout-${threadId}.jsonl`),
+ [
+ JSON.stringify({
+ type: "session_meta",
+ payload: {
+ id: threadId,
+ ...(parent === undefined ? {} : { parent_thread_id: parent }),
+ },
+ }),
+ JSON.stringify({
+ type: "event_msg",
+ payload: {
+ type: "token_count",
+ info: { total_token_usage: usage },
+ },
+ }),
+ "",
+ ].join("\n"),
+ );
+}
+
+test("the paired evaluator preserves aggregate usage across production completion and failure", async (context) => {
+ const root = await temporaryDirectory("time-budget-production-");
+ context.after(() => rm(root, { recursive: true, force: true }));
+ const pluginRoot = fileURLToPath(
+ new URL("../../plugins/codex-security/", import.meta.url),
+ );
+ const prepared = await prepareStandardEval(root, pluginRoot);
+ const outputRoot = join(root, "results");
+ await mkdir(outputRoot);
+ await writeFile(
+ join(outputRoot, "fixture.json"),
+ JSON.stringify(prepared.fixture),
+ );
+ const states: string[] = [];
+ const commands: string[] = [];
+ const prompts: string[] = [];
+ const records = await runComparison({
+ prepared,
+ config: { model: "gpt-5.6-sol", pairs: 2, budgetMinutes: 5 },
+ run: createStandardRunner({
+ stateRoot: join(root, "states"),
+ outputRoot,
+ environment: { OPENAI_API_KEY: "synthetic-environment" },
+ createClient: (config, dependencies) => {
+ const home = dependencies.environment["CODEX_SECURITY_STATE_DIR"]!;
+ states.push(home);
+ const client = new CodexSecurity(
+ config,
+ {
+ ...dependencies,
+ now: () => Date.UTC(2026, 0, 1),
+ probeCodexSandbox: async () => {},
+ prepareScanArtifactRestorer: async () => ({
+ restore: async () => {},
+ }),
+ prepareRuntime: async () => ({
+ codexHome: home,
+ plugin: {
+ pluginRoot,
+ marketplaceRoot: pluginRoot,
+ installedRoot: pluginRoot,
+ marketplaceName: "codex-security-sdk",
+ name: "codex-security",
+ version: "0.1.0",
+ },
+ environment: {},
+ credentialsAvailable: true,
+ }),
+ resolvePluginPython: async () => "/managed/python",
+ repositoryRevision: async () => "deadbeef",
+ runWorkbench: async (_options, args): Promise<JsonObject> => {
+ commands.push(args[0]!);
+ if (
+ args[0] === "prepare-scan-completion" &&
+ states.length === 1
+ ) {
+ throw new Error(
+ "Synthetic completion failure after model usage",
+ );
+ }
+ if (args[0] === "register-cli-scan") {
+ return {
+ scanId: "scan_example_001",
+ targetId: "target_sha256_example",
+ targetRevision: "deadbeef",
+ scanDir: args[args.indexOf("--scan-dir") + 1]!,
+ contract: { target: { allowedKinds: ["git_revision"] } },
+ };
+ }
+ if (args[0] === "get-scan-feedback") {
+ return {
+ scanId: "scan_example_001",
+ targetId: "target_sha256_example",
+ falsePositives: [],
+ };
+ }
+ return {};
+ },
+ },
+ { surface: "sdk" },
+ );
+ return {
+ run: client.run.bind(client),
+ close: async () => {
+ await client.close();
+ if (states.length <= 2)
+ throw new Error("Synthetic cleanup failure");
+ },
+ };
+ },
+ createCodex: async (options) => ({
+ startThread: (threadOptions) => ({
+ id: "thread-1",
+ runStreamed: async (prompt) => {
+ prompts.push(prompt);
+ const scanDir = options.env!["CODEX_SECURITY_SCAN_DIR"]!;
+ const home = options.env!["CODEX_HOME"]!;
+ const nearby = join(scanDir, "..", "..");
+ assert.equal(threadOptions?.workingDirectory, scanDir);
+ for (const path of [
+ scanDir,
+ home,
+ options.env!["CODEX_SECURITY_STATE_DIR"]!,
+ ]) {
+ assert.doesNotMatch(relative(root, path), /baseline|time-aware/u);
+ }
+ assert.equal(existsSync(join(nearby, "fixture.json")), false);
+ assert.equal(
+ existsSync(join(nearby, "pair-1-time-aware", "result.json")),
+ false,
+ );
+ for (const previous of states.slice(0, -1)) {
+ assert.equal(existsSync(previous), false);
+ }
+ await writeUsageSession(home, "thread-1", {
+ input_tokens: 100,
+ output_tokens: 20,
+ });
+ await writeUsageSession(
+ home,
+ "worker-thread",
+ { input_tokens: 50, output_tokens: 7 },
+ "thread-1",
+ );
+ await cp(join(pluginRoot, "examples", "completed-scan"), scanDir, {
+ recursive: true,
+ });
+ await writeFile(
+ join(scanDir, "report.md"),
+ "# Finalized synthetic report\n",
+ );
+ async function* events(): AsyncGenerator<ThreadEvent> {
+ yield { type: "thread.started", thread_id: "thread-1" };
+ yield { type: "turn.started" };
+ yield {
+ type: "turn.completed",
+ usage: {
+ input_tokens: 10,
+ cached_input_tokens: 2,
+ cache_write_input_tokens: 0,
+ output_tokens: 3,
+ reasoning_output_tokens: 1,
+ },
+ };
+ }
+ return { events: events() };
+ },
+ }),
+ }),
+ }),
+ onResult: async (record, result) => {
+ assert.equal(existsSync(states.at(-1)!), false);
+ if (result !== undefined) {
+ await writeFile(
+ join(
+ outputRoot,
+ `pair-${record.pair}-${record.variant}`,
+ "result.json",
+ ),
+ JSON.stringify(result),
+ );
+ }
+ },
+ });
+ assert.equal(new Set(states).size, 4);
+ assert.equal(prompts.length, 4, JSON.stringify(records));
+ assert.ok(!prompts[0].includes("# Investigation time allowance"));
+ assert.ok(prompts[1].includes("2026-01-01T00:05:00.000Z"));
+ assert.deepEqual(
+ records.map((record) => record.status),
+ ["failed", "failed", "completed", "completed"],
+ JSON.stringify(records),
+ );
+ assert.match(
+ records[0].error!,
+ /Synthetic completion failure after model usage/,
+ );
+ assert.match(records[0].error!, /Synthetic cleanup failure/);
+ assert.equal(records[1].error, "Synthetic cleanup failure");
+ assert.equal(
+ commands.filter((command) => command === "prepare-scan-completion").length,
+ 4,
+ );
+ assert.equal(
+ commands.filter((command) => command === "complete-scan").length,
+ 3,
+ );
+ for (const [index, record] of records.entries()) {
+ assert.partialDeepStrictEqual(record.usage, {
+ input_tokens: 150,
+ output_tokens: 27,
+ });
+ const archivedReport = join(
+ outputRoot,
+ `pair-${record.pair}-${record.variant}`,
+ "scan",
+ "report.md",
+ );
+ assert.match(
+ await readFile(archivedReport, "utf8"),
+ /Finalized synthetic report/,
+ );
+ if (record.status === "completed")
+ assert.equal(record.report.reportPath, archivedReport);
+ assert.equal(record.promptSha256, hashText(prompts[index]));
+ }
+ const summary = summarizeRuns(records);
+ assert.equal(summary.arms.baseline.usage.total_tokens, 354);
+ assert.equal(summary.paired.completedPairs, 1);
+ assert.equal(summary.paired.excludedPairs, 1);
+});
diff --git a/evals/standard-time-budget/test_summary.mts b/evals/standard-time-budget/test_summary.mts
new file mode 100644
index 000000000..8ea3678d5
--- /dev/null
+++ b/evals/standard-time-budget/test_summary.mts
@@ -0,0 +1,189 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { summarizeRuns } from "./summary.mts";
+import { createFixture } from "./fixtures.mts";
+import { gradeResult } from "./grade.mts";
+import type { CompletedRunRecord, RunReport, Variant } from "./experiment.mts";
+
+function attempt(pair: number, variant: Variant) {
+ return {
+ pair,
+ variant,
+ elapsedMs: 100,
+ startedAt: "2026-01-01T00:00:00.000Z",
+ advisoryBudgetMinutes: variant === "time-aware" ? 5 : null,
+ promptSha256: null,
+ usage: null,
+ };
+}
+
+function completed(
+ pair: number,
+ variant: Variant,
+ overrides: Partial<RunReport> = {},
+): CompletedRunRecord {
+ return {
+ ...attempt(pair, variant),
+ status: "completed",
+ usage: { input_tokens: 100, cached_input_tokens: 40, output_tokens: 20 },
+ report: {
+ ...gradeResult({ findings: [] }, createFixture()),
+ recall: 0.5,
+ foundCount: 1,
+ expectedCount: 2,
+ safeControlFindings: 0,
+ unmatchedFindings: [],
+ evidenceErrors: [],
+ ...overrides,
+ },
+ };
+}
+
+test("pairs runs by identifier and reports time-aware minus baseline", () => {
+ const result = summarizeRuns([
+ completed(2, "time-aware", { recall: 1, foundCount: 2 }),
+ completed(1, "baseline"),
+ completed(2, "baseline"),
+ {
+ ...completed(1, "time-aware", {
+ recall: 0,
+ foundCount: 0,
+ safeControlFindings: 1,
+ unmatchedFindings: [
+ { findingIndex: 0, title: "unmatched", reason: "unknown source" },
+ ],
+ evidenceErrors: [
+ { reason: "bad source" },
+ { reason: "missing evidence" },
+ ],
+ }),
+ elapsedMs: 300,
+ },
+ ]);
+ assert.equal(result.arms.baseline.attempted, 2);
+ assert.equal(result.arms.baseline.successRate, 1);
+ assert.equal(result.arms["time-aware"].meanElapsedMs, 200);
+ assert.equal(result.arms["time-aware"].meanRecall, 0.5);
+ assert.equal(result.arms["time-aware"].foundCount, 2);
+ assert.equal(result.arms["time-aware"].expectedCount, 4);
+ assert.equal(result.arms["time-aware"].safeControlFindings, 1);
+ assert.equal(result.arms["time-aware"].unmatchedFindings, 1);
+ assert.equal(result.arms["time-aware"].evidenceErrors, 2);
+ assert.equal(result.paired.completedPairs, 2);
+ assert.equal(result.paired.excludedPairs, 0);
+ assert.equal(result.paired.meanDelta.recall, 0);
+ assert.equal(result.paired.meanDelta.elapsedMs, 100);
+ assert.equal(result.paired.meanDelta.safeControlFindings, 0.5);
+ assert.equal(result.paired.meanDelta.evidenceErrors, 1);
+});
+
+test("shows failures and missing arms alongside completed-pair improvements", () => {
+ const result = summarizeRuns([
+ completed(1, "baseline"),
+ completed(1, "time-aware", { recall: 1, foundCount: 2 }),
+ completed(2, "baseline"),
+ {
+ ...attempt(2, "time-aware"),
+ status: "failed",
+ error: "scan failed",
+ elapsedMs: 900,
+ },
+ completed(3, "baseline"),
+ ]);
+ assert.equal(result.arms["time-aware"].attempted, 2);
+ assert.equal(result.arms["time-aware"].completed, 1);
+ assert.equal(result.arms["time-aware"].failed, 1);
+ assert.equal(result.arms["time-aware"].successRate, 0.5);
+ assert.equal(result.arms["time-aware"].meanElapsedMs, 500);
+ assert.equal(result.arms["time-aware"].meanRecall, 1);
+ assert.equal(result.paired.totalPairs, 3);
+ assert.equal(result.paired.completedPairs, 1);
+ assert.equal(result.paired.excludedPairs, 2);
+ assert.equal(result.paired.meanDelta.recall, 0.5);
+ assert.equal(result.arms["time-aware"].usage.input_tokens, null);
+ assert.equal(result.arms["time-aware"].usage.total_tokens, null);
+ assert.equal(result.arms["time-aware"].usage.missingUsageRuns, 1);
+});
+
+test("cached input tokens are a subset and never added to total tokens", () => {
+ const result = summarizeRuns([
+ completed(1, "baseline"),
+ {
+ ...completed(1, "time-aware"),
+ usage: { input_tokens: 150, cached_input_tokens: 80, output_tokens: 40 },
+ },
+ ]);
+ assert.deepEqual(result.arms.baseline.usage, {
+ input_tokens: 100,
+ cached_input_tokens: 40,
+ output_tokens: 20,
+ total_tokens: 120,
+ missingUsageRuns: 0,
+ });
+ assert.equal(result.paired.meanDelta.input_tokens, 50);
+ assert.equal(result.paired.meanDelta.cached_input_tokens, 40);
+ assert.equal(result.paired.meanDelta.output_tokens, 20);
+ assert.equal(result.paired.meanDelta.total_tokens, 70);
+});
+
+test("missing usage remains unknown without discarding completed pairs", () => {
+ const result = summarizeRuns([
+ completed(1, "baseline"),
+ { ...completed(1, "time-aware"), usage: null },
+ completed(2, "baseline"),
+ completed(2, "time-aware"),
+ ]);
+ assert.equal(result.arms["time-aware"].usage.input_tokens, null);
+ assert.equal(result.arms["time-aware"].usage.cached_input_tokens, null);
+ assert.equal(result.arms["time-aware"].usage.output_tokens, null);
+ assert.equal(result.arms["time-aware"].usage.total_tokens, null);
+ assert.equal(result.arms["time-aware"].usage.missingUsageRuns, 1);
+ assert.equal(result.paired.completedPairs, 2);
+ assert.equal(result.paired.meanDelta.total_tokens, null);
+ assert.equal(result.paired.meanDelta.recall, 0);
+});
+
+test("a missing cache count does not hide known input and output totals", () => {
+ const result = summarizeRuns([
+ {
+ ...completed(1, "baseline"),
+ usage: { input_tokens: 100, output_tokens: 20 },
+ },
+ ]);
+ assert.equal(result.arms.baseline.usage.input_tokens, 100);
+ assert.equal(result.arms.baseline.usage.cached_input_tokens, null);
+ assert.equal(result.arms.baseline.usage.total_tokens, 120);
+ assert.equal(result.arms.baseline.usage.missingUsageRuns, 1);
+});
+
+test("empty and entirely failed arms have no measured quality or deltas", () => {
+ const empty = summarizeRuns([]);
+ assert.equal(empty.arms.baseline.attempted, 0);
+ assert.equal(empty.arms.baseline.successRate, null);
+ assert.equal(empty.arms.baseline.meanElapsedMs, null);
+ assert.equal(empty.arms.baseline.usage.total_tokens, null);
+ assert.equal(empty.paired.totalPairs, 0);
+ assert.equal(empty.paired.meanDelta.recall, null);
+ const failed = summarizeRuns([
+ { ...attempt(1, "baseline"), status: "failed", error: "scan failed" },
+ {
+ ...attempt(1, "time-aware"),
+ status: "failed",
+ error: "scan failed",
+ elapsedMs: 200,
+ },
+ ]);
+ assert.equal(failed.arms.baseline.successRate, 0);
+ assert.equal(failed.arms.baseline.meanRecall, null);
+ assert.equal(failed.arms.baseline.safeControlFindings, null);
+ assert.equal(failed.paired.completedPairs, 0);
+ assert.equal(failed.paired.excludedPairs, 1);
+ assert.equal(failed.paired.meanDelta.elapsedMs, null);
+});
+
+test("duplicate arms cannot silently bias results", () => {
+ assert.throws(
+ () => summarizeRuns([completed(1, "baseline"), completed(1, "baseline")]),
+ /Duplicate run for pair 1\/baseline/,
+ );
+});
diff --git a/evals/tsconfig.json b/evals/tsconfig.json
index 354d2a787..18228790b 100644
--- a/evals/tsconfig.json
+++ b/evals/tsconfig.json
@@ -1,12 +1,14 @@
{
"extends": "../plugins/codex-security/mcp-app/tsconfig.json",
"compilerOptions": {
+ "lib": ["esnext", "dom", "dom.iterable"],
"typeRoots": ["../sdk/typescript/node_modules/@types"]
},
"include": [
"../plugins/codex-security/mcp-app/src/assets.d.ts",
"deep-reducer/*.mts",
"secret-discovery/*.mts",
+ "standard-time-budget/*.mts",
"workflow-routing/*.mts",
"triage-finding/assertions/*.mts",
"triage-finding/scripts/*.mts",
diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json
index 28f773f5f..b59fec017 100644
--- a/sdk/typescript/package.json
+++ b/sdk/typescript/package.json
@@ -66,7 +66,7 @@
"build:plugin": "node scripts/build-plugin.mjs",
"check:plugin-source": "node scripts/check-plugin-source.mjs",
"check:package": "node scripts/check-package.mjs",
- "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.mts\" \"../../plugins/codex-security/native/*.mts\" \"../../evals/{deep-reducer,secret-discovery,workflow-routing}/*.mts\" \"../../evals/triage-finding/{assertions,scripts,sastbench/assertions,sastbench/scripts}/*.mts\" \"../../evals/triage-finding/types.ts\" \"../../evals/tsconfig.json\" && node scripts/check-markdown.mjs",
+ "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.mts\" \"../../plugins/codex-security/native/*.mts\" \"../../evals/{deep-reducer,secret-discovery,standard-time-budget,workflow-routing}/*.mts\" \"../../evals/triage-finding/{assertions,scripts,sastbench/assertions,sastbench/scripts}/*.mts\" \"../../evals/triage-finding/types.ts\" \"../../evals/tsconfig.json\" && node scripts/check-markdown.mjs",
"generate:models": "node scripts/generate-models.cjs",
"generate:models:check": "node scripts/generate-models.cjs --check",
"generate:external-import-models": "node scripts/generate-external-import-models.cjs",
diff --git a/sdk/typescript/tests-ts/scan-directory-identity.test.ts b/sdk/typescript/tests-ts/scan-directory-identity.test.ts
new file mode 100644
index 000000000..3fc882ad3
--- /dev/null
+++ b/sdk/typescript/tests-ts/scan-directory-identity.test.ts
@@ -0,0 +1,214 @@
+import { afterEach, expect, spyOn, test } from "bun:test";
+import {
+ chmod,
+ cp,
+ mkdir,
+ readFile,
+ rename,
+ rm,
+ writeFile,
+ type FileHandle,
+} from "node:fs/promises";
+import type { BigIntStats, Stats } from "node:fs";
+import { join } from "node:path";
+import { loadContract, readScanFile } from "../src/contract.js";
+import { ContractValidationError } from "../src/errors.js";
+import { PLUGIN_ROOT } from "./plugin-root.js";
+import { createApiTestFixtures } from "./support/temporary-directories.js";
+import { runTestInSubprocess } from "./support/test-subprocess.js";
+
+const { temporaryDirectory, cleanup } = createApiTestFixtures(
+ "codex-security-directory-identity-",
+);
+afterEach(cleanup);
+const scenarios = [
+ "initial-root",
+ "accepted-root",
+ "between-documents",
+ "parent-before-leaf",
+ "parent-retains-leaf",
+] as const;
+const identities = [
+ ["small-distinct", 100n, 101n],
+ ["rounded-distinct", 2n ** 60n, 2n ** 60n + 1n],
+ ["unchanged", 2n ** 60n, 2n ** 60n],
+] as const;
+
+for (const scenario of scenarios) {
+ for (const [kind, originalId, replacementId] of identities) {
+ const title = `checked scan directory ${scenario} preserves ${kind} identities`;
+ test(title, async () => {
+ if (runTestInSubprocess(import.meta.path, title)) return;
+ const root = await temporaryDirectory();
+ const scanDir = join(root, "scan");
+ const isParent = scenario.startsWith("parent-");
+ const swappedPath = isParent ? join(scanDir, "artifacts") : scanDir;
+ const replacementPath = join(root, "replacement");
+ const backupPath = join(root, "original");
+ const leaf = join(swappedPath, "document.json");
+ if (isParent) {
+ await mkdir(swappedPath, { recursive: true, mode: 0o700 });
+ await chmod(scanDir, 0o700);
+ await mkdir(replacementPath, { mode: 0o700 });
+ await writeFile(leaf, "original data");
+ await writeFile(
+ join(replacementPath, "document.json"),
+ "replacement data",
+ );
+ } else {
+ await cp(join(PLUGIN_ROOT, "examples", "completed-scan"), scanDir, {
+ recursive: true,
+ });
+ await cp(
+ join(PLUGIN_ROOT, "examples", "completed-scan"),
+ replacementPath,
+ { recursive: true },
+ );
+ await chmod(scanDir, 0o700);
+ await chmod(replacementPath, 0o700);
+ for (const [directory, name] of [
+ [scanDir, "original contract"],
+ [replacementPath, "replacement contract"],
+ ]) {
+ const path = join(directory!, "scan-manifest.json");
+ const manifest = JSON.parse(await readFile(path, "utf8"));
+ manifest.scan.target.displayName = name;
+ await writeFile(path, JSON.stringify(manifest));
+ }
+ }
+ const files = await import("node:fs/promises");
+ const originalLstat = files.lstat;
+ const originalRealpath = files.realpath;
+ const originalOpen = files.open;
+ const runtime = await import("../src/runtime.js");
+ const originalAncestry = runtime.requireSecureOutputAncestry;
+ const originalDirectory = await originalLstat(swappedPath, {
+ bigint: true,
+ });
+ const replacementDirectory = await originalLstat(replacementPath, {
+ bigint: true,
+ });
+ expect(originalDirectory.ino).not.toBe(replacementDirectory.ino);
+ const originalLeaf = isParent
+ ? await originalLstat(leaf, { bigint: true })
+ : undefined;
+ let swapped = false;
+ const handles: FileHandle[] = [];
+ const restoreCloses: Array<() => void> = [];
+ const swap = async () => {
+ if (swapped || kind === "unchanged") return;
+ await rename(swappedPath, backupPath);
+ await rename(replacementPath, swappedPath);
+ if (scenario === "parent-retains-leaf") {
+ await rm(leaf);
+ await rename(join(backupPath, "document.json"), leaf);
+ }
+ swapped = true;
+ };
+ const withIdentity = (metadata: Stats | BigIntStats) => {
+ Object.defineProperty(metadata, "ino", {
+ value:
+ typeof metadata.ino === "bigint"
+ ? swapped
+ ? replacementId
+ : originalId
+ : Number(swapped ? replacementId : originalId),
+ });
+ return metadata;
+ };
+ const inspecting = spyOn(files, "lstat").mockImplementation((async (
+ ...args: Parameters<typeof originalLstat>
+ ) => {
+ if (scenario === "parent-before-leaf" && args[0] === leaf) await swap();
+ const metadata = await originalLstat(...args);
+ return args[0] === swappedPath ? withIdentity(metadata) : metadata;
+ }) as typeof originalLstat);
+ const resolving = spyOn(files, "realpath").mockImplementation((async (
+ ...args: Parameters<typeof originalRealpath>
+ ) => {
+ const canonical = await originalRealpath(...args);
+ if (
+ (scenario === "initial-root" && args[0] === scanDir) ||
+ (scenario === "parent-retains-leaf" && args[0] === leaf)
+ )
+ await swap();
+ return canonical;
+ }) as typeof originalRealpath);
+ const ancestry = spyOn(
+ runtime,
+ "requireSecureOutputAncestry",
+ ).mockImplementation(async (...args) => {
+ await originalAncestry(...args);
+ if (scenario === "accepted-root" && args[0] === scanDir) await swap();
+ });
+ const opening = spyOn(files, "open").mockImplementation(
+ async (...args) => {
+ const handle = await originalOpen(...args);
+ handles.push(handle);
+ if (
+ scenario === "between-documents" &&
+ args[0] === join(scanDir, "scan-manifest.json")
+ ) {
+ const close = handle.close.bind(handle);
+ const closing = spyOn(handle, "close").mockImplementation(
+ async () => {
+ await close();
+ await swap();
+ },
+ );
+ restoreCloses.push(() => closing.mockRestore());
+ }
+ return handle;
+ },
+ );
+ let value: string | undefined;
+ let error: unknown;
+ try {
+ try {
+ value = isParent
+ ? (
+ await readScanFile(
+ scanDir,
+ "artifacts/document.json",
+ "fixture",
+ )
+ ).toString()
+ : (
+ await loadContract(scanDir, {
+ pluginRoot: PLUGIN_ROOT,
+ expectedScanId: "scan_example_001",
+ })
+ ).manifest.scan.target.displayName;
+ } catch (caught) {
+ error = caught;
+ }
+ const finalDirectory = await originalLstat(swappedPath, {
+ bigint: true,
+ });
+ const finalLeaf = isParent
+ ? await originalLstat(leaf, { bigint: true })
+ : undefined;
+ expect(swapped).toBe(kind !== "unchanged");
+ for (const handle of handles) expect(handle.fd).toBe(-1);
+ if (isParent || scenario === "between-documents")
+ expect(handles.length).toBeGreaterThan(0);
+ if (kind === "unchanged") {
+ expect(finalDirectory.ino).toBe(originalDirectory.ino);
+ expect(error).toBeUndefined();
+ expect(value).toBe(isParent ? "original data" : "original contract");
+ } else {
+ expect(finalDirectory.ino).toBe(replacementDirectory.ino);
+ expect(error).toBeInstanceOf(ContractValidationError);
+ }
+ if (scenario === "parent-retains-leaf")
+ expect(finalLeaf?.ino).toBe(originalLeaf?.ino);
+ } finally {
+ inspecting.mockRestore();
+ resolving.mockRestore();
+ opening.mockRestore();
+ ancestry.mockRestore();
+ for (const restore of restoreCloses) restore();
+ }
+ });
+ }
+}