OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(evals): compare production Standard scan time budgets (#1679)
Michael D'Angelo32b24b5d7cdc
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.ymlindex 9a47bae56..c37f0da58 100644--- a/.github/workflows/node-ci.yml+++ b/.github/workflows/node-ci.yml@@ -462,6 +462,14 @@ jobs:- name: Test secret discovery eval contractsif: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'run: node --experimental-strip-types --test evals/secret-discovery/test_*.mts+ - name: Build host plugin for Standard eval integration+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'+ run: |+ node plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs --output plugins/codex-security/mcp --native host+ ln -s ../../plugins/codex-security sdk/typescript/_bundled_plugin+ - name: Test Standard time-budget eval contracts+ if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'+ run: node --experimental-strip-types --test evals/standard-time-budget/test_*.mts- name: Test workflow routing eval contractsif: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'run: node --experimental-strip-types --test evals/workflow-routing/test_*.mtsdiff --git a/evals/README.md b/evals/README.mdindex 7a4ad74c2..2e3fa50bb 100644--- a/evals/README.md+++ b/evals/README.md@@ -11,10 +11,13 @@ being embedded in its source tree or shipped npm runtime.- [Secret discovery](secret-discovery/README.md): checks whether the productioncore audit finds synthetic credentials in source and keeps them in its finalfindings, with deterministic grading and harness checks.+- [Standard time budget](standard-time-budget/README.md): paired comparison of+ production Standard scans with and without an advisory time allowance on+ synthetic vulnerabilities and safe controls, with source-evidence grading.- [Workflow routing](workflow-routing/README.md): checks whether repository-installedproduction skills route natural-language requests to the appropriate workflow,stopping before workflow execution.-Model runs are opt-in. CI runs the deterministic triage, secret-discovery, and-workflow-routing helper checks and the real-IPC reducer regression through the-normal MCP test suite.+Model runs are opt-in. CI runs the deterministic triage, secret-discovery,+time-budget, and workflow-routing helper checks and the real-IPC reducer+regression through the normal MCP test suite.diff --git a/evals/secret-discovery/fixtures.mts b/evals/secret-discovery/fixtures.mtsindex 5d90cf2ce..4a55a1463 100644--- a/evals/secret-discovery/fixtures.mts+++ b/evals/secret-discovery/fixtures.mts@@ -165,7 +165,7 @@ CONTENT_SHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcd/** Stage source only. Gold labels and secret-value lookup stay in the harness. */export async function writeFixture(repo: string,- fixture: ReturnType<typeof createFixture>,+ fixture: Pick<ReturnType<typeof createFixture>, "files">,) {for (const [relative, contents] of Object.entries(fixture.files)) {const destination = join(repo, relative);diff --git a/evals/secret-discovery/runtime.mts b/evals/secret-discovery/runtime.mtsindex f2956e1fa..8cdb8b3b8 100644--- a/evals/secret-discovery/runtime.mts+++ b/evals/secret-discovery/runtime.mts@@ -24,7 +24,7 @@ export { bundledCodexSdkEnvironment } from "../../sdk/typescript/src/codex-sdk-e/** Keep temporary source and the login link alive until cancelled work stops. */export async function withEvalState(createHome: (parent?: string) => Promise<string>,- ambientHome: string,+ ambientHome: string | undefined,run: (state: {home: string;root: string;@@ -47,11 +47,13 @@ export async function withEvalState(root = await realpath(await mkdtemp(join(tmpdir(), "source-audit-")));// Share saved login updates without importing the caller's configuration.let auth;- try {- const path = await realpath(join(ambientHome, "auth.json"));- if ((await stat(path)).isFile()) auth = path;- } catch (error) {- if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;+ if (ambientHome !== undefined) {+ try {+ const path = await realpath(join(ambientHome, "auth.json"));+ if ((await stat(path)).isFile()) auth = path;+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;+ }}// A hard link must stay on the saved login's filesystem, including symlinks.const allocatedHome = await createHome(auth ? dirname(auth) : undefined);diff --git a/evals/standard-time-budget/.gitignore b/evals/standard-time-budget/.gitignorenew file mode 100644index 000000000..30846059d--- /dev/null+++ b/evals/standard-time-budget/.gitignore@@ -0,0 +1 @@+/reports/diff --git a/evals/standard-time-budget/README.md b/evals/standard-time-budget/README.mdnew file mode 100644index 000000000..2a2417af5--- /dev/null+++ b/evals/standard-time-budget/README.md@@ -0,0 +1,118 @@+# Standard Scan time allowance comparison++Compare production Standard scans with and without `maxTimeMinutes`. Both+arms call `CodexSecurity.run`, including setup, delegation, canonical artifact+validation, report generation, and completion. The treatment uses the shared+[investigation-time-budget.md](../../plugins/codex-security/references/investigation-time-budget.md)+policy, not an eval-only prompt.++The SDK stamps one shared UTC deadline after setup, immediately before the+first scan turn. The allowance is advisory: there is no abort timer, automatic+continuation, or minimum runtime. Finalization is outside the allowance.++## Run++Use the repository's dependency and+[native-artifact setup](../../plugins/codex-security/native/README.md), then run+from the repository root:++```sh+pnpm --dir sdk/typescript run build:plugin+pnpm --dir sdk/typescript run build:evals+node --experimental-strip-types evals/standard-time-budget/run.mts+```++Rebuild the bundle after plugin changes. This developer eval requires an existing+`OPENAI_API_KEY` or `CODEX_API_KEY` environment variable; saved Codex or+`codex-security login` credentials are intentionally unsupported. It uses the+bundled Codex SDK without reading saved logins, provisioning credentials, or+changing persistent configuration. Production SDK and CLI login support is+unchanged. **Model runs consume usage and have no enforced time or spend cap.**++[experiment.json](experiment.json) selects an explicit model, two pairs+(four scans including delegated work), and five advisory minutes. Both arms+use `xhigh` reasoning and production worker settings. Pairs alternate+baseline-first and treatment-first order. Override settings with a JSON file:++```sh+node --experimental-strip-types evals/standard-time-budget/run.mts /path/to/experiment.json+```++The fields are `model`, `pairs`, and `budgetMinutes`; positive fractional+minutes are accepted. Production deadline validation runs before either arm+starts. Ctrl+C saves the interrupted attempt, stops future arms, and waits for+child processes before cleaning temporary state.++## Fixture and isolation++The synthetic application has tenant authorization, SQL injection, file+traversal, and reflected HTML bugs, each with a safe control. Source and+deployment context go to the model; gold labels and expected locations stay+in the evaluator.++Each arm runs in an opaque temporary directory outside the artifact tree; pair+and variant labels appear only in evaluator reports and archives. After the+client closes, its scan output is archived and its state is removed before the+next arm. If state removal fails, the failed attempt is recorded and the+comparison stops before another arm starts. Gold labels and prior-arm results+are not next to the model's working directory or included in prompts. This is layout separation, not a filesystem+security boundary: production API-key authentication and root-read permissions+remain unchanged. This eval does not test resumption or external publication.++## Results++Artifacts are saved under ignored `reports/`, updated after every attempt:++- `experiment.json`: settings, evaluator/shared-helper, source, full-plugin,+ and built-SDK file hashes, plus SDK/Codex package versions and the selected+ Codex executable paths and SHA-256 hashes, including a `CODEX_CLI_PATH` override.+ npm launchers also record their native executable. Generated reports+ are excluded from evaluator hashes.+- `fixture.json`: synthetic source and held-out gold labels for review.+- `pair-<number>-<variant>/prompt.txt`: the actual first dispatched scan prompt.+- Per-arm `scan/`, `result.json`, and `report.json`: canonical scan artifacts,+ semantic output, grading, usage, elapsed time, and exact failure text.+- `runs.json` and `summary.json`: all attempts and the paired comparison.++Elapsed time is monotonic and includes SDK setup, investigation, finalization,+scan archival, cleanup, and grading, but not subsequent evaluator JSON writing.+A canonical scan that misses a bug still completed successfully. The runner+exits nonzero for execution failures, not low recall.++The grader counts unique gold cases with matching CWE, a `root_control` or+`sink` source location, and authentic source evidence. Other or missing roles+do not earn automatic recall; unmatched findings require adjudication. It+consumes the production contract's repository-relative POSIX paths. Duplicates+do not increase recall.+Safe-control overlaps, unmatched findings, and citation errors are separate+signals; overlaps and unmatched findings require adjudication, not automatic+false-positive labels. Free-text location roles can flag benign context+citations. Preserve raw scores and record adjudication separately: source+matching is a heuristic, not independent exploit validation.++The summary separates execution success from completed-run quality. Paired+deltas are treatment minus baseline and include only complete pairs; excluded+pairs remain explicit. Each attempt's top-level `usage` includes delegated+workers, even when completion or grading fails. Missing usage stays unknown,+cached input is a subset of input tokens, and no dollar cost is inferred.++## Interpretation and checks++Two pairs are a smoke comparison, not statistical evidence. Review findings+without variant labels and compare grounded recall, adjudicated false+positives, completion, tokens, and elapsed time together. More findings or a+longer run alone is not an improvement, and these arms do not have matched+spend. Expand the corpus if both arms saturate the fixture; use more pairs and+pinned public repository revisions before drawing quality/cost conclusions.++After the builds above, run deterministic checks without model calls:++```sh+node --experimental-strip-types --test evals/standard-time-budget/test_*.mts evals/secret-discovery/test_*.mts+```++The integration test uses the real SDK and canonical artifact helpers with+mocked model/workbench boundaries, verifying isolated state, completion, prompt+capture, and parent-plus-worker usage. CI reuses its host native build, builds+the plugin with `build_mcp_app.mjs --native host`, and links it at+`sdk/typescript/_bundled_plugin`; universal release artifacts are not required.diff --git a/evals/standard-time-budget/experiment.json b/evals/standard-time-budget/experiment.jsonnew file mode 100644index 000000000..dd10dd2a3--- /dev/null+++ b/evals/standard-time-budget/experiment.json@@ -0,0 +1,5 @@+{+ "model": "gpt-5.6-sol",+ "pairs": 2,+ "budgetMinutes": 5+}diff --git a/evals/standard-time-budget/experiment.mts b/evals/standard-time-budget/experiment.mtsnew file mode 100644index 000000000..397fa62a5--- /dev/null+++ b/evals/standard-time-budget/experiment.mts@@ -0,0 +1,171 @@+import { createHash } from "node:crypto";+import { readdir, readFile } from "node:fs/promises";+import { join } from "node:path";+import type { GradeReport, Result } from "./grade.mts";+import type { PreparedStandardEval } from "./fixtures.mts";+import type { StandardRunner, StandardRunOptions } from "./production.mts";++export interface ExperimentConfig {+ model: string;+ pairs: number;+ budgetMinutes: number;+}+export type Variant = "baseline" | "time-aware";+/** Retained arm state must not be exposed to a later arm. */+export class StateCleanupError extends AggregateError {}+export type RunReport = GradeReport & {+ scanId?: string;+ scanDir?: string;+ reportPath?: string;+};+interface RunMetadata {+ pair: number;+ variant: Variant;+ startedAt: string;+ advisoryBudgetMinutes: number | null;+ promptSha256: string | null;+ elapsedMs: number;+ usage: unknown;+}+type RunOutcome =+ | { status: "completed"; report: RunReport; error?: never }+ | { status: "failed"; error: string; report?: never };+export type RunRecord = RunMetadata & RunOutcome;+export type CompletedRunRecord = Extract<RunRecord, { status: "completed" }>;+interface ComparisonOptions {+ prepared: PreparedStandardEval;+ config: ExperimentConfig;+ signal?: AbortSignal;+ run: StandardRunner;+ elapsedNow?: () => number;+ onStart?: (record: Pick<RunMetadata, "pair" | "variant">) => Promise<unknown>;+ onResult?: (+ record: RunRecord,+ result: Result | undefined,+ records: RunRecord[],+ prompt: string | undefined,+ ) => Promise<unknown>;+}++export function validateExperiment(config: ExperimentConfig) {+ if (typeof config?.model !== "string" || !config.model.trim()) {+ throw new Error("The experiment requires an explicit model.");+ }+ if (!Number.isSafeInteger(config.pairs) || config.pairs < 1) {+ throw new Error("Experiment pairs must be a positive integer.");+ }+ if (!Number.isFinite(config.budgetMinutes) || config.budgetMinutes <= 0) {+ throw new Error("Experiment budgetMinutes must be a positive number.");+ }+ return {+ model: config.model,+ pairs: config.pairs,+ budgetMinutes: config.budgetMinutes,+ };+}++export function hashText(text: string | Uint8Array) {+ return createHash("sha256").update(text).digest("hex");+}++export async function snapshotEvaluator(directory = import.meta.dirname) {+ const files: Record<string, string> = {};+ for (const path of [+ "run.mts",+ "production.mts",+ "experiment.mts",+ "fixtures.mts",+ "grade.mts",+ "summary.mts",+ "../secret-discovery/fixtures.mts",+ "../secret-discovery/runtime.mts",+ ]) {+ files[path] = hashText(await readFile(join(directory, path)));+ }+ return files;+}++export async function snapshotFiles(directory: string, prefix = "") {+ const files: Record<string, string> = {};+ const entries = await readdir(directory, { withFileTypes: true });+ for (const entry of entries.sort((first, second) =>+ first.name.localeCompare(second.name),+ )) {+ const relative = prefix ? `${prefix}/${entry.name}` : entry.name;+ const path = join(directory, entry.name);+ if (entry.isDirectory()) {+ Object.assign(files, await snapshotFiles(path, relative));+ } else {+ files[relative] = hashText(await readFile(path));+ }+ }+ return files;+}++export async function runComparison({+ prepared,+ config,+ signal,+ run,+ elapsedNow = () => performance.now(),+ onStart = async () => {},+ onResult = async () => {},+}: ComparisonOptions) {+ config = validateExperiment(config);+ const records: RunRecord[] = [];+ for (let pair = 1; pair <= config.pairs; pair += 1) {+ const variants: Variant[] =+ pair % 2 === 1 ? ["baseline", "time-aware"] : ["time-aware", "baseline"];+ for (const variant of variants) {+ signal?.throwIfAborted();+ await onStart({ pair, variant });+ signal?.throwIfAborted();+ const startedAt = new Date().toISOString();+ const startedElapsed = elapsedNow();+ const observation: StandardRunOptions["observation"] = {};+ let outcome: RunOutcome;+ let semanticResult: Result | undefined;+ let cleanupFailure: StateCleanupError | undefined;+ try {+ const result = await run(prepared, {+ model: config.model,+ pair,+ variant,+ signal,+ ...(variant === "time-aware"+ ? { maxTimeMinutes: config.budgetMinutes }+ : {}),+ observation,+ });+ outcome = { status: "completed", report: result.report };+ semanticResult = result.semanticResult;+ } catch (error) {+ if (error instanceof StateCleanupError) cleanupFailure = error;+ outcome = {+ status: "failed",+ error: error instanceof Error ? error.message : String(error),+ };+ }+ const elapsedMs = elapsedNow() - startedElapsed;+ const record: RunRecord = {+ pair,+ variant,+ startedAt,+ advisoryBudgetMinutes:+ variant === "time-aware" ? config.budgetMinutes : null,+ promptSha256:+ observation.prompt === undefined+ ? null+ : hashText(observation.prompt),+ ...outcome,+ elapsedMs,+ usage: observation.usage ?? null,+ };+ records.push(record);+ await onResult(record, semanticResult, records, observation.prompt);+ if (cleanupFailure !== undefined) throw cleanupFailure;+ signal?.throwIfAborted();+ }+ }+ return records;+}diff --git a/evals/standard-time-budget/fixtures.mts b/evals/standard-time-budget/fixtures.mtsnew file mode 100644index 000000000..590fa8bb0--- /dev/null+++ b/evals/standard-time-budget/fixtures.mts@@ -0,0 +1,254 @@+export interface SourceCase {+ id: string;+ path: string;+ line: number;+ cwes: string[];+}++export interface Fixture {+ files: Record<string, string>;+ positives: SourceCase[];+ controls: SourceCase[];+}++export function createFixture(): Fixture {+ const files: Record<string, string> = {+ "README.md": `# Account portal++This Python Flask service serves authenticated tenant accounts. Both the account+and billing routes are currently supported. The sign-in service creates signed+Flask sessions containing a server-selected integer tenant_id; clients cannot+choose that value. The session signing key comes from deployment configuration.++Invoices and catalog records belong to exactly one tenant. Account users may+read only their own tenant's records. Invoice identifiers are sequential.++Export downloads are available to all authenticated users, but only files under+/srv/portal/exports may be downloaded. That directory and its symlinks are+deployment-managed; account users cannot create or modify files there. Other+server files, including configuration, must remain private.++Preview endpoints render user-provided titles in HTML for the signed-in browser.+The deployment does not add an HTML sanitizer or Content-Security-Policy header.+The database and export files are deployment resources, not checked-in fixtures.+`,+ "src/app.py": `import os+import sqlite3++from flask import Flask, abort, g, session++from .downloads import downloads+from .invoices import invoices+from .preview import preview+from .search import search++app = Flask(__name__)+app.secret_key = os.environ["SESSION_SIGNING_KEY"]+app.config.update(SESSION_COOKIE_SECURE=True, SESSION_COOKIE_HTTPONLY=True,+ SESSION_COOKIE_SAMESITE="Lax")++[email protected]_request+def authenticate():+ tenant_id = session.get("tenant_id")+ if type(tenant_id) is not int:+ abort(401)+ g.tenant_id = tenant_id+ g.database = sqlite3.connect(os.environ["PORTAL_DATABASE"])+ g.database.row_factory = sqlite3.Row++[email protected]_request+def close_database(error):+ database = g.pop("database", None)+ if database is not None:+ database.close()+++app.register_blueprint(invoices)+app.register_blueprint(search)+app.register_blueprint(downloads)+app.register_blueprint(preview)+`,+ "src/__init__.py": "",+ "src/invoices.py": `from flask import Blueprint, abort, g, jsonify++invoices = Blueprint("invoices", __name__)++[email protected]("/account/invoices/<int:invoice_id>")+def account_invoice(invoice_id):+ row = g.database.execute(+ "SELECT id, tenant_id, recipient, total FROM invoices WHERE id = ?",+ (invoice_id,),+ ).fetchone()+ if row is None:+ abort(404)+ return jsonify(dict(row))++[email protected]("/billing/invoices/<int:invoice_id>")+def billing_invoice(invoice_id):+ row = g.database.execute(+ "SELECT id, tenant_id, recipient, total FROM invoices WHERE id = ? AND tenant_id = ?",+ (invoice_id, g.tenant_id),+ ).fetchone()+ if row is None:+ abort(404)+ return jsonify(dict(row))+`,+ "src/search.py": `from flask import Blueprint, g, jsonify, request++search = Blueprint("search", __name__)++[email protected]("/account/search")+def account_search():+ term = request.args.get("q", "")+ query = f"SELECT id, name FROM catalog WHERE tenant_id = {g.tenant_id} AND name LIKE '%{term}%'"+ rows = g.database.execute(query).fetchall()+ return jsonify([dict(row) for row in rows])++[email protected]("/billing/search")+def billing_search():+ term = request.args.get("q", "")+ rows = g.database.execute(+ "SELECT id, name FROM catalog WHERE tenant_id = ? AND name LIKE ?",+ (g.tenant_id, f"%{term}%"),+ ).fetchall()+ return jsonify([dict(row) for row in rows])+`,+ "src/downloads.py": `from pathlib import Path++from flask import Blueprint, abort, request, send_file++downloads = Blueprint("downloads", __name__)+EXPORT_ROOT = Path("/srv/portal/exports")++[email protected]("/account/download")+def account_download():+ name = request.args["name"]+ return send_file(EXPORT_ROOT / name, as_attachment=True)++[email protected]("/billing/download")+def billing_download():+ name = request.args["name"]+ root = EXPORT_ROOT.resolve()+ target = (root / name).resolve()+ if not target.is_relative_to(root):+ abort(403)+ return send_file(target, as_attachment=True)+`,+ "src/preview.py": `from html import escape++from flask import Blueprint, Response, request++preview = Blueprint("preview", __name__)++[email protected]("/account/preview")+def account_preview():+ title = request.args.get("title", "")+ return Response(f"<h1>{title}</h1>", content_type="text/html")++[email protected]("/billing/preview")+def billing_preview():+ title = escape(request.args.get("title", ""))+ return Response(f"<h1>{title}</h1>", content_type="text/html")+`,+ "schema.sql": `CREATE TABLE invoices (+ id INTEGER PRIMARY KEY,+ tenant_id INTEGER NOT NULL,+ recipient TEXT NOT NULL,+ total INTEGER NOT NULL+);++CREATE TABLE catalog (+ id INTEGER PRIMARY KEY,+ tenant_id INTEGER NOT NULL,+ name TEXT NOT NULL+);+`,+ };+ const sourceCase = (+ id: string,+ path: string,+ marker: string,+ cwes: string[],+ occurrence = 0,+ ): SourceCase => {+ const matchingLines = files[path]+ .split("\n")+ .flatMap((line, index) => (line.includes(marker) ? [index + 1] : []));+ const line = matchingLines[occurrence];+ if (!line) throw new Error(`Fixture marker missing: ${id}`);+ return { id, path, line, cwes };+ };+ return {+ files,+ positives: [+ sourceCase(+ "invoice-tenant-isolation",+ "src/invoices.py",+ 'FROM invoices WHERE id = ?"',+ ["CWE-639", "CWE-862", "CWE-863", "CWE-284"],+ ),+ sourceCase("catalog-sql-injection", "src/search.py", "query = f", [+ "CWE-89",+ ]),+ sourceCase(+ "export-path-traversal",+ "src/downloads.py",+ "send_file(EXPORT_ROOT",+ ["CWE-22", "CWE-23", "CWE-36", "CWE-73"],+ ),+ sourceCase(+ "preview-reflected-html",+ "src/preview.py",+ "return Response",+ ["CWE-79"],+ ),+ ],+ controls: [+ sourceCase(+ "invoice-scoped-query",+ "src/invoices.py",+ "AND tenant_id = ?",+ ["CWE-639", "CWE-862", "CWE-863", "CWE-284"],+ ),+ sourceCase("catalog-bound-parameters", "src/search.py", "name LIKE ?", [+ "CWE-89",+ ]),+ sourceCase(+ "export-contained-path",+ "src/downloads.py",+ "send_file(target",+ ["CWE-22", "CWE-23", "CWE-36", "CWE-73"],+ ),+ sourceCase(+ "preview-escaped-html",+ "src/preview.py",+ "return Response",+ ["CWE-79"],+ 1,+ ),+ ],+ };+}+import { join } from "node:path";+import { writeFixture } from "../secret-discovery/fixtures.mts";++export async function prepareStandardEval(root: string, runtime: string) {+ const fixture = createFixture();+ const repo = join(root, "repository");+ await writeFixture(repo, fixture);+ return { repo, runtime, fixture };+}++export type PreparedStandardEval = Awaited<+ ReturnType<typeof prepareStandardEval>+>;diff --git a/evals/standard-time-budget/grade.mts b/evals/standard-time-budget/grade.mtsnew file mode 100644index 000000000..985782d97--- /dev/null+++ b/evals/standard-time-budget/grade.mts@@ -0,0 +1,205 @@+import { posix } from "node:path";+import type {+ Finding as ScanFinding,+ FindingCodeEvidence,+ FindingLocation,+} from "../../sdk/typescript/src/models.js";+import type { Fixture, SourceCase } from "./fixtures.mts";++export type SourceLocation = FindingLocation;+type Finding = Pick<ScanFinding, "title" | "locations"> & {+ taxonomy: Pick<ScanFinding["taxonomy"], "cwe">;+ codeEvidence?: Pick<FindingCodeEvidence, "path" | "startLine" | "code">[];+};++export interface Result {+ findings: Finding[];+ [key: string]: unknown;+}++interface EvidenceError {+ findingIndex?: number;+ evidenceIndex?: number;+ locationIndex?: number;+ caseId?: string;+ reason: string;+}++export type GradeReport = ReturnType<typeof gradeResult>;++function validRange(location: SourceLocation, fixture: Fixture) {+ const source = fixture.files[posix.normalize(location.path)];+ const endLine = location.endLine ?? location.startLine;+ return (+ typeof source === "string" &&+ Number.isInteger(location.startLine) &&+ Number.isInteger(endLine) &&+ location.startLine > 0 &&+ endLine >= location.startLine &&+ endLine <= source.trimEnd().split("\n").length+ );+}++function overlaps(location: SourceLocation, expected: SourceCase) {+ return (+ posix.normalize(location.path) === expected.path &&+ location.startLine <= expected.line &&+ (location.endLine ?? location.startLine) >= expected.line+ );+}++function matches(+ finding: Finding,+ expected: SourceCase,+ fixture: Fixture,+ requirePrimaryRole: boolean,+) {+ return (+ finding.taxonomy.cwe.some((cwe) =>+ expected.cwes.includes(String(cwe).trim().toUpperCase()),+ ) &&+ finding.locations.some(+ (location) =>+ (requirePrimaryRole+ ? /^(root_control|sink)$/u.test(location.role ?? "")+ : !/^(supporting|support|context|consumer|expected_control)$/u.test(+ location.role ?? "",+ )) &&+ validRange(location, fixture) &&+ overlaps(location, expected),+ )+ );+}++function inspectEvidence(+ finding: Finding,+ fixture: Fixture,+ findingIndex: number,+) {+ const entries = finding.codeEvidence ?? [];+ const ranges: SourceLocation[] = [];+ const errors: EvidenceError[] = [];+ if (!entries.length) {+ errors.push({ findingIndex, reason: "missing code evidence" });+ }+ for (const [evidenceIndex, entry] of entries.entries()) {+ const code = entry.code.replaceAll("\r\n", "\n").replace(/\n$/u, "");+ const range = {+ ...entry,+ endLine: entry.startLine + code.split("\n").length - 1,+ };+ const source = fixture.files[posix.normalize(entry.path)];+ if (+ !code.trim() ||+ !validRange(range, fixture) ||+ code !==+ source!+ .split("\n")+ .slice(entry.startLine - 1, range.endLine)+ .join("\n")+ ) {+ errors.push({+ findingIndex,+ evidenceIndex,+ reason: "code does not match source at the cited lines",+ });+ } else {+ ranges.push(range);+ }+ }+ return { ranges, errors };+}++export function gradeResult(result: Result, fixture: Fixture) {+ const { findings } = result;+ const evidenceErrors: EvidenceError[] = [];+ const unmatchedFindings = [];+ const safeControlDetails = [];+ const matchCounts = new Map(+ fixture.positives.map((expected) => [expected.id, 0]),+ );+ for (const [findingIndex, finding] of findings.entries()) {+ const evidence = inspectEvidence(finding, fixture, findingIndex);+ for (const [locationIndex, location] of finding.locations.entries()) {+ if (!validRange(location, fixture)) {+ evidence.errors.push({+ findingIndex,+ locationIndex,+ reason: "location path or line range does not exist in the source",+ });+ }+ }+ evidenceErrors.push(...evidence.errors);+ const candidates = fixture.positives.filter((expected) =>+ matches(finding, expected, fixture, true),+ );+ const matched = candidates.filter((expected) => {+ const grounded = evidence.ranges.some((range) =>+ overlaps(range, expected),+ );+ if (!grounded) {+ evidenceErrors.push({+ findingIndex,+ caseId: expected.id,+ reason: "code evidence does not cover the expected source line",+ });+ }+ return grounded && evidence.errors.length === 0;+ });+ for (const expected of matched) {+ matchCounts.set(expected.id, matchCounts.get(expected.id)! + 1);+ }+ const controls = fixture.controls.filter((expected) =>+ matches(finding, expected, fixture, false),+ );+ if (controls.length) {+ safeControlDetails.push({+ findingIndex,+ title: finding.title,+ controlIds: controls.map((entry) => entry.id),+ });+ }+ if (!matched.length && !controls.length) {+ unmatchedFindings.push({+ findingIndex,+ title: finding.title,+ reason: candidates.length+ ? "expected case lacks authentic evidence"+ : "no matching source location and CWE in the gold cases",+ });+ }+ }+ const cases = fixture.positives.map((expected) => ({+ id: expected.id,+ found: matchCounts.get(expected.id)! > 0,+ }));+ const matchedIds = cases+ .filter((entry) => entry.found)+ .map((entry) => entry.id);+ const foundCount = matchedIds.length;+ const expectedCount = cases.length;+ return {+ passed:+ foundCount === expectedCount &&+ safeControlDetails.length === 0 &&+ unmatchedFindings.length === 0 &&+ evidenceErrors.length === 0,+ recall: foundCount / expectedCount,+ foundCount,+ expectedCount,+ matchedIds,+ cases,+ duplicateFindings: [...matchCounts.values()].reduce(+ (total, count) => total + Math.max(0, count - 1),+ 0,+ ),+ safeControlFindings: safeControlDetails.length,+ safeControlDetails,+ unmatchedFindings,+ evidenceErrors,+ needsAdjudication:+ unmatchedFindings.length > 0 || safeControlDetails.length > 0,+ gradingMethod:+ "Heuristic source-line, CWE, and verbatim evidence matching; not independent exploit validation. Unmatched findings require manual adjudication and are not automatically false positives.",+ };+}diff --git a/evals/standard-time-budget/production.mts b/evals/standard-time-budget/production.mtsnew file mode 100644index 000000000..ab634b358--- /dev/null+++ b/evals/standard-time-budget/production.mts@@ -0,0 +1,183 @@+import { randomUUID } from "node:crypto";+import { cp, mkdir, readFile, rm } from "node:fs/promises";+import { join, relative } from "node:path";+import { Codex } from "../../sdk/typescript/node_modules/@openai/codex-sdk/dist/index.js";+import * as sdk from "../../sdk/typescript/dist/api.js";+import { deepScanOptions } from "../../sdk/typescript/dist/deep-config.js";+import { createProfileCodex } from "../../sdk/typescript/dist/provider-profile.js";+import {+ codexExecutableReadPaths,+ resolveCodexCommand,+} from "../../sdk/typescript/dist/runtime.js";+import type { CodexSecurity as Client } from "../../sdk/typescript/src/api.js";+import type { PreparedStandardEval } from "./fixtures.mts";+import { gradeResult, type Result } from "./grade.mts";+import {+ hashText,+ StateCleanupError,+ type RunReport,+ type Variant,+} from "./experiment.mts";++const { CodexSecurity, scanAuthentication } =+ sdk as unknown as typeof import("../../sdk/typescript/src/api.js");+type Dependencies = NonNullable<ConstructorParameters<typeof CodexSecurity>[1]>;++export async function snapshotCodexExecutable(environment: NodeJS.ProcessEnv) {+ const command = resolveCodexCommand(environment).command;+ const paths = await codexExecutableReadPaths(command);+ return {+ command,+ files: await Promise.all(+ paths.map(async (path) => ({+ path,+ sha256: hashText(await readFile(path)),+ })),+ ),+ };+}++export function preflightStandardEval(+ environment: NodeJS.ProcessEnv,+ budgetMinutes: number,+) {+ if (scanAuthentication(environment).method !== "api_key") {+ throw new Error(+ "The Standard time-budget eval requires OPENAI_API_KEY or CODEX_API_KEY. Saved logins are not supported by this developer eval.",+ );+ }+ deepScanOptions({ mode: "standard", maxTimeMinutes: budgetMinutes });+}++export interface StandardRunOptions {+ model: string;+ pair: number;+ variant: Variant;+ maxTimeMinutes?: number;+ signal?: AbortSignal;+ observation: { prompt?: string; usage?: unknown };+}++export type StandardRunner = (+ prepared: PreparedStandardEval,+ options: StandardRunOptions,+) => Promise<{ report: RunReport; semanticResult: Result }>;++export function createStandardRunner({+ stateRoot,+ outputRoot,+ environment = process.env,+ createClient = (config, dependencies) =>+ new CodexSecurity(config, dependencies, { surface: "sdk" }),+ createCodex = ({ nativeProfile, ...settings }) =>+ nativeProfile === undefined+ ? new Codex(settings)+ : createProfileCodex(settings, nativeProfile),+}: {+ stateRoot: string;+ outputRoot: string;+ environment?: NodeJS.ProcessEnv;+ createClient?: (+ config: ConstructorParameters<typeof CodexSecurity>[0],+ dependencies: Dependencies,+ ) => Pick<Client, "run" | "close">;+ createCodex?: Dependencies["createCodex"];+}): StandardRunner {+ return async (prepared, options) => {+ const stateDirectory = join(stateRoot, randomUUID());+ const scanDirectory = join(stateDirectory, "scan");+ const archiveDirectory = join(+ outputRoot,+ `pair-${options.pair}-${options.variant}`,+ "scan",+ );+ await mkdir(scanDirectory, { recursive: true, mode: 0o700 });+ const client = createClient(+ {+ pluginPath: prepared.runtime,+ codexOverrides: {+ model: options.model,+ model_reasoning_effort: "xhigh",+ },+ },+ {+ environment: {+ ...environment,+ CODEX_SECURITY_STATE_DIR: stateDirectory,+ },+ createCodex: async (settings) => {+ const codex = await createCodex(settings);+ return {+ startThread: (threadOptions) => {+ const thread = codex.startThread(threadOptions);+ return {+ get id() {+ return thread.id;+ },+ runStreamed: (prompt, turnOptions) => {+ options.observation.prompt ??= prompt;+ return thread.runStreamed(prompt, turnOptions);+ },+ };+ },+ };+ },+ },+ );+ let failure: unknown;+ try {+ const result = await client.run(prepared.repo, {+ mode: "standard",+ maxTimeMinutes: options.maxTimeMinutes,+ signal: options.signal,+ onUsage: (usage) => {+ options.observation.usage = usage;+ },+ outputDir: scanDirectory,+ });+ options.observation.usage =+ result.turnResult.usage ?? options.observation.usage;+ const semanticResult = {+ findings: result.findings.findings,+ coverage: result.coverage,+ threatModel: result.threatModel,+ };+ return {+ semanticResult,+ report: {+ ...gradeResult(semanticResult, prepared.fixture),+ scanId: result.manifest.scan.id,+ scanDir: archiveDirectory,+ reportPath: join(+ archiveDirectory,+ relative(result.scanDir, result.reportPath),+ ),+ },+ };+ } catch (error) {+ failure = error;+ throw error;+ } finally {+ const failures: unknown[] = failure === undefined ? [] : [failure];+ try {+ await client.close();+ } catch (error) {+ failures.push(error);+ }+ try {+ await cp(scanDirectory, archiveDirectory, { recursive: true });+ } catch (error) {+ failures.push(error);+ }+ try {+ await rm(stateDirectory, { recursive: true, force: true });+ } catch (error) {+ failures.push(error);+ throw new StateCleanupError(failures, failures.map(String).join("\n"));+ }+ if (failures.length > 1)+ throw new AggregateError(failures, failures.map(String).join("\n"));+ if (failures.length === 1 && failure === undefined) throw failures[0];+ }+ };+}diff --git a/evals/standard-time-budget/run.mts b/evals/standard-time-budget/run.mtsnew file mode 100644index 000000000..64b27a8c9--- /dev/null+++ b/evals/standard-time-budget/run.mts@@ -0,0 +1,112 @@+import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import {+ bundledPluginRoot,+ createIsolatedHome,+} from "../../sdk/typescript/dist/runtime.js";+import {+ CODEX_EXECUTABLE_VERSION,+ CODEX_SDK_VERSION,+ VERSION,+} from "../../sdk/typescript/dist/version.js";+import { withEvalState } from "../secret-discovery/runtime.mts";+import {+ createStandardRunner,+ preflightStandardEval,+ snapshotCodexExecutable,+} from "./production.mts";+import { prepareStandardEval } from "./fixtures.mts";+import {+ runComparison,+ snapshotEvaluator,+ snapshotFiles,+ validateExperiment,+} from "./experiment.mts";+import { summarizeRuns } from "./summary.mts";++const config = validateExperiment(+ JSON.parse(+ await readFile(+ process.argv[2] ?? new URL("./experiment.json", import.meta.url),+ "utf8",+ ),+ ),+);+preflightStandardEval(process.env, config.budgetMinutes);+const codexExecutable = await snapshotCodexExecutable(process.env);+const pluginRoot = await bundledPluginRoot();+const reports = fileURLToPath(new URL("./reports/", import.meta.url));+await mkdir(reports, { recursive: true });+const reportDirectory = await mkdtemp(join(reports, "run-"));+const saveJson = (path: string, value: unknown) =>+ writeFile(path, JSON.stringify(value, null, 2) + "\n");+console.log(`Eval artifacts: ${reportDirectory}`);+await withEvalState(+ createIsolatedHome,+ undefined,+ async ({ root, home, signal }) => {+ const prepared = await prepareStandardEval(root, pluginRoot);+ await saveJson(join(reportDirectory, "experiment.json"), {+ ...config,+ reasoningEffort: "xhigh",+ execution: "CodexSecurity.run Standard production pipeline",+ order: "alternating baseline-first and time-aware-first pairs",+ evaluator: await snapshotEvaluator(),+ source: await snapshotFiles(prepared.repo),+ plugin: await snapshotFiles(pluginRoot),+ sdk: await snapshotFiles(+ fileURLToPath(new URL("../../sdk/typescript/dist/", import.meta.url)),+ ),+ packageVersions: {+ "@openai/codex-security": VERSION,+ "@openai/codex-sdk": CODEX_SDK_VERSION,+ "@openai/codex": CODEX_EXECUTABLE_VERSION,+ },+ codexExecutable,+ });+ await saveJson(join(reportDirectory, "fixture.json"), prepared.fixture);+ const records = await runComparison({+ prepared,+ config,+ signal,+ run: createStandardRunner({+ stateRoot: join(root, "states"),+ outputRoot: reportDirectory,+ environment: { ...process.env, CODEX_HOME: home },+ }),+ onStart: async (record) => {+ const directory = join(+ reportDirectory,+ `pair-${record.pair}-${record.variant}`,+ );+ await mkdir(directory);+ console.log(`Pair ${record.pair}/${config.pairs}: ${record.variant}`);+ },+ onResult: async (record, semanticResult, completedRecords, prompt) => {+ const directory = join(+ reportDirectory,+ `pair-${record.pair}-${record.variant}`,+ );+ if (prompt !== undefined)+ await writeFile(join(directory, "prompt.txt"), prompt);+ if (semanticResult !== undefined) {+ await saveJson(join(directory, "result.json"), semanticResult);+ }+ await saveJson(join(directory, "report.json"), record);+ await saveJson(join(reportDirectory, "runs.json"), completedRecords);+ await saveJson(+ join(reportDirectory, "summary.json"),+ summarizeRuns(completedRecords),+ );+ console.log(+ `${record.variant}: ${record.status}, ${(record.elapsedMs / 1000).toFixed(1)}s`,+ );+ },+ });+ console.log(JSON.stringify(summarizeRuns(records), null, 2));+ process.exitCode = records.some((record) => record.status === "failed")+ ? 1+ : 0;+ },+);diff --git a/evals/standard-time-budget/summary.mts b/evals/standard-time-budget/summary.mtsnew file mode 100644index 000000000..45a8c18ab--- /dev/null+++ b/evals/standard-time-budget/summary.mts@@ -0,0 +1,143 @@+import type {+ CompletedRunRecord,+ RunRecord,+ RunReport,+ Variant,+} from "./experiment.mts";++const variants = ["baseline", "time-aware"] as const;+const usageFields = [+ "input_tokens",+ "cached_input_tokens",+ "output_tokens",+] as const;+type UsageField = (typeof usageFields)[number];++function mean(values: number[]) {+ return values.length+ ? values.reduce((total, value) => total + value, 0) / values.length+ : null;+}++function usageValue(record: RunRecord, field: UsageField) {+ const usage = record.usage as Partial<Record<UsageField, number>> | undefined;+ const value = usage?.[field];+ return typeof value === "number" && Number.isFinite(value) ? value : null;+}++function totalTokens(record: RunRecord) {+ const input = usageValue(record, "input_tokens");+ const output = usageValue(record, "output_tokens");+ return input === null || output === null ? null : input + output;+}++function sumKnown(values: (number | null)[]) {+ return values.length &&+ values.every((value): value is number => value !== null)+ ? values.reduce((total, value) => total + value, 0)+ : null;+}++function summarizeArm(records: RunRecord[]) {+ const completed = records.filter(+ (record): record is CompletedRunRecord => record.status === "completed",+ );+ const qualityCount = (getValue: (report: RunReport) => number) =>+ completed.length+ ? completed.reduce((total, record) => total + getValue(record.report), 0)+ : null;+ return {+ attempted: records.length,+ completed: completed.length,+ failed: records.length - completed.length,+ successRate: records.length ? completed.length / records.length : null,+ meanElapsedMs: mean(records.map((record) => record.elapsedMs)),+ meanRecall: mean(completed.map((record) => record.report.recall)),+ foundCount: qualityCount((report) => report.foundCount),+ expectedCount: qualityCount((report) => report.expectedCount),+ safeControlFindings: qualityCount((report) => report.safeControlFindings),+ unmatchedFindings: qualityCount(+ (report) => report.unmatchedFindings.length,+ ),+ evidenceErrors: qualityCount((report) => report.evidenceErrors.length),+ usage: {+ ...(Object.fromEntries(+ usageFields.map((field) => [+ field,+ sumKnown(records.map((record) => usageValue(record, field))),+ ]),+ ) as Record<UsageField, number | null>),+ total_tokens: sumKnown(records.map(totalTokens)),+ missingUsageRuns: records.filter((record) =>+ usageFields.some((field) => usageValue(record, field) === null),+ ).length,+ },+ };+}++export function summarizeRuns(records: RunRecord[]) {+ const pairs = new Map<number, Partial<Record<Variant, RunRecord>>>();+ for (const record of records) {+ const variant = record.variant;+ const pair = pairs.get(record.pair) ?? {};+ if (pair[variant]) {+ throw new Error(+ `Duplicate run for pair ${record.pair}/${record.variant}`,+ );+ }+ pair[variant] = record;+ pairs.set(record.pair, pair);+ }+ const completedPairs = [...pairs.values()].filter(+ (pair): pair is Record<Variant, CompletedRunRecord> =>+ variants.every((variant) => pair[variant]?.status === "completed"),+ );+ const meanDelta = (+ getValue: (record: CompletedRunRecord) => number | null,+ ) => {+ const deltas = completedPairs.map((pair) => {+ const baseline = getValue(pair.baseline);+ const timeAware = getValue(pair["time-aware"]);+ return baseline === null || timeAware === null+ ? null+ : timeAware - baseline;+ });+ return deltas.every((delta): delta is number => delta !== null)+ ? mean(deltas)+ : null;+ };+ return {+ arms: Object.fromEntries(+ variants.map((variant) => [+ variant,+ summarizeArm(records.filter((record) => record.variant === variant)),+ ]),+ ),+ paired: {+ totalPairs: pairs.size,+ completedPairs: completedPairs.length,+ excludedPairs: pairs.size - completedPairs.length,+ meanDelta: {+ recall: meanDelta((record) => record.report.recall),+ elapsedMs: meanDelta((record) => record.elapsedMs),+ foundCount: meanDelta((record) => record.report.foundCount),+ safeControlFindings: meanDelta(+ (record) => record.report.safeControlFindings,+ ),+ unmatchedFindings: meanDelta(+ (record) => record.report.unmatchedFindings.length,+ ),+ evidenceErrors: meanDelta(+ (record) => record.report.evidenceErrors.length,+ ),+ ...(Object.fromEntries(+ usageFields.map((field) => [+ field,+ meanDelta((record) => usageValue(record, field)),+ ]),+ ) as Record<UsageField, number | null>),+ total_tokens: meanDelta(totalTokens),+ },+ },+ };+}diff --git a/evals/standard-time-budget/test_experiment.mts b/evals/standard-time-budget/test_experiment.mtsnew file mode 100644index 000000000..97583bc6d--- /dev/null+++ b/evals/standard-time-budget/test_experiment.mts@@ -0,0 +1,204 @@+import assert from "node:assert/strict";+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import test from "node:test";+import { writeFixture } from "../secret-discovery/fixtures.mts";+import {+ createFixture,+ prepareStandardEval,+ type PreparedStandardEval,+} from "./fixtures.mts";+import { gradeResult, type Result } from "./grade.mts";+import {+ hashText,+ runComparison,+ snapshotEvaluator,+ snapshotFiles,+ validateExperiment,+ type RunRecord,+} from "./experiment.mts";++const config = { model: "eval-model", pairs: 2, budgetMinutes: 5 };+const preparedStub: PreparedStandardEval = {+ repo: "/synthetic/repo",+ runtime: "/synthetic/runtime",+ fixture: createFixture(),+};++function mockResult() {+ return {+ report: {+ ...gradeResult({ findings: [] }, preparedStub.fixture),+ },+ semanticResult: { findings: [] },+ };+}++test("evaluator provenance tracks implementation and shared helpers, not reports", async (context) => {+ const root = await mkdtemp(join(tmpdir(), "time-budget-provenance-"));+ context.after(() => rm(root, { recursive: true, force: true }));+ const directory = join(root, "standard-time-budget");+ const files = Object.fromEntries(+ [+ "run.mts",+ "production.mts",+ "experiment.mts",+ "fixtures.mts",+ "grade.mts",+ "summary.mts",+ "../secret-discovery/fixtures.mts",+ "../secret-discovery/runtime.mts",+ ].map((path) => [path, path]),+ );+ await writeFixture(directory, { files });+ const before = await snapshotEvaluator(directory);+ assert.deepEqual(+ before,+ Object.fromEntries(+ Object.entries(files).map(([path, text]) => [path, hashText(text)]),+ ),+ );+ await writeFixture(directory, {+ files: {+ "grade.mts": "changed grader",+ "../secret-discovery/runtime.mts": "changed state helper",+ "reports/run-123/experiment.json": "generated report",+ },+ });+ assert.deepEqual(await snapshotEvaluator(directory), {+ ...before,+ "grade.mts": hashText("changed grader"),+ "../secret-discovery/runtime.mts": hashText("changed state helper"),+ });+});++test("pairs stage source without labels, alternate order, and opt in only for treatment", async (context) => {+ const root = await mkdtemp(join(tmpdir(), "time-budget-eval-"));+ context.after(() => rm(root, { recursive: true, force: true }));+ const prepared = await prepareStandardEval(root, "/synthetic/plugin");+ const sourceBefore = await snapshotFiles(prepared.repo);+ assert.deepEqual(+ Object.keys(sourceBefore).sort(),+ Object.keys(prepared.fixture.files).sort(),+ );+ const requests: (number | undefined)[] = [];+ const records = await runComparison({+ prepared,+ config,+ run: async (actual, options) => {+ assert.equal(actual, prepared);+ requests.push(options.maxTimeMinutes);+ options.observation.prompt = "Captured production prompt";+ return mockResult();+ },+ });+ assert.deepEqual(requests, [undefined, 5, 5, undefined]);+ assert.deepEqual(+ records.map(({ pair, variant }) => [pair, variant]),+ [+ [1, "baseline"],+ [1, "time-aware"],+ [2, "time-aware"],+ [2, "baseline"],+ ],+ );+ assert.equal(records[0].promptSha256, hashText("Captured production prompt"));+});++test("monotonic elapsed time includes runner work but not artifact callbacks", async () => {+ let elapsed = 0;+ let calls = 0;+ const records = await runComparison({+ prepared: preparedStub,+ config: { ...config, pairs: 1 },+ elapsedNow: () => elapsed,+ onStart: async () => {+ elapsed += 60_000;+ },+ run: async () => {+ calls += 1;+ elapsed += calls === 1 ? 1_000 : 400_000;+ return mockResult();+ },+ onResult: async () => {+ elapsed += 60_000;+ },+ });+ assert.equal(calls, 2);+ assert.deepEqual(+ records.map((record) => record.elapsedMs),+ [1_000, 400_000],+ );+ assert.equal(records[1].status, "completed");+});++test("retains exact failures and still runs the paired arm", async () => {+ const saved: {+ record: RunRecord;+ result: Result | undefined;+ count: number;+ }[] = [];+ let calls = 0;+ const records = await runComparison({+ prepared: preparedStub,+ config: { ...config, pairs: 1 },+ run: async () => {+ calls += 1;+ if (calls === 1)+ throw new Error("model stream failed: synthetic diagnostic");+ return mockResult();+ },+ onResult: async (record, result, records) =>+ saved.push({ record, result, count: records.length }),+ });+ assert.equal(records[0].status, "failed");+ assert.equal(records[0].error, "model stream failed: synthetic diagnostic");+ assert.equal(records[0].promptSha256, null);+ assert.equal(records[1].status, "completed");+ assert.deepEqual(+ saved.map(({ count }) => count),+ [1, 2],+ );+ assert.equal(saved[0].result, undefined);+});++test("explicit cancellation saves the interrupted attempt and stops the comparison", async () => {+ const controller = new AbortController();+ const saved: RunRecord[] = [];+ let calls = 0;+ await assert.rejects(+ runComparison({+ prepared: preparedStub,+ config,+ signal: controller.signal,+ run: async (_prepared, { signal }) => {+ calls += 1;+ controller.abort(new Error("user canceled"));+ signal!.throwIfAborted();+ throw new Error("Cancellation should abort the runner");+ },+ onResult: async (record) => saved.push(record),+ }),+ /user canceled/,+ );+ assert.equal(calls, 1);+ assert.equal(saved.length, 1);+ assert.equal(saved[0].status, "failed");+});++test("requires an explicit model and valid experiment sizes without imposing a cap", () => {+ for (const change of [+ { model: "" },+ { pairs: 0 },+ { pairs: 1.5 },+ { budgetMinutes: 0 },+ { budgetMinutes: Infinity },+ ]) {+ assert.throws(() => validateExperiment({ ...config, ...change }));+ }+ assert.deepEqual(+ validateExperiment({ ...config, pairs: 100, budgetMinutes: 0.25 }),+ { ...config, pairs: 100, budgetMinutes: 0.25 },+ );+});diff --git a/evals/standard-time-budget/test_grading.mts b/evals/standard-time-budget/test_grading.mtsnew file mode 100644index 000000000..a98ad1e41--- /dev/null+++ b/evals/standard-time-budget/test_grading.mts@@ -0,0 +1,230 @@+import assert from "node:assert/strict";+import test from "node:test";+import { createFixture, type Fixture, type SourceCase } from "./fixtures.mts";+import { gradeResult, type SourceLocation } from "./grade.mts";++function findingFor(fixture: Fixture, expected: SourceCase) {+ return {+ title: expected.id,+ taxonomy: { cwe: [expected.cwes[0]] },+ locations: [+ { path: expected.path, startLine: expected.line, role: "root_control" },+ ] as SourceLocation[],+ codeEvidence: [+ {+ path: expected.path,+ startLine: expected.line,+ code: fixture.files[expected.path].split("\n")[expected.line - 1],+ },+ ],+ };+}++type FixtureFinding = ReturnType<typeof findingFor>;++function completeResult(fixture: Fixture) {+ return {+ findings: fixture.positives.map((expected) =>+ findingFor(fixture, expected),+ ),+ };+}++test("authentic source evidence matches all four vulnerability families", () => {+ const fixture = createFixture();+ const result = completeResult(fixture);+ result.findings[1].locations[0].role = "sink";+ const report = gradeResult(result, fixture);+ assert.equal(report.passed, true);+ assert.equal(report.recall, 1);+ assert.equal(report.expectedCount, 4);+ assert.equal(report.foundCount, 4);+ assert.deepEqual(report.evidenceErrors, []);+});++test("a missed bug reduces recall", () => {+ const fixture = createFixture();+ const result = completeResult(fixture);+ result.findings.pop();+ const report = gradeResult(result, fixture);+ assert.equal(report.recall, 0.75);+ assert.equal(report.passed, false);+ assert.deepEqual(report.unmatchedFindings, []);+});++test("additional fabricated locations invalidate an otherwise grounded finding", () => {+ const fixture = createFixture();+ for (const role of ["root_control", "supporting"]) {+ for (const location of [+ { path: "src/invented.py", startLine: 500 },+ { path: "src/invoices.py", startLine: 500 },+ { path: "src/invoices.py", startLine: 0 },+ { path: "src/invoices.py", startLine: 8, endLine: 7 },+ ]) {+ const result = completeResult(fixture);+ result.findings[0].locations.push({ ...location, role });+ const report = gradeResult(result, fixture);+ assert.equal(report.passed, false);+ assert.equal(report.recall, 0.75);+ assert.equal(report.needsAdjudication, true);+ assert.deepEqual(report.evidenceErrors, [+ {+ findingIndex: 0,+ locationIndex: 1,+ reason: "location path or line range does not exist in the source",+ },+ ]);+ }+ }+});++test("duplicate reports cannot increase recall", () => {+ const fixture = createFixture();+ const result = completeResult(fixture);+ result.findings.push(structuredClone(result.findings[0]));+ const report = gradeResult(result, fixture);+ assert.equal(report.recall, 1);+ assert.equal(report.foundCount, 4);+ assert.equal(report.duplicateFindings, 1);+});++test("a different CWE, source line, or non-primary role does not earn a gold match", () => {+ const fixture = createFixture();+ const alterations: Array<(finding: FixtureFinding) => void> = [+ (finding) => {+ finding.taxonomy.cwe = ["CWE-79"];+ },+ (finding) => {+ finding.locations[0].path = "README.md";+ },+ (finding) => {+ finding.locations[0].startLine = 1;+ },+ (finding) => {+ finding.locations[0].role = "supporting";+ },+ (finding) => {+ finding.locations[0].role = "supporting_context";+ finding.locations.unshift({+ path: "README.md",+ startLine: 1,+ role: "root_control",+ });+ },+ ];+ for (const alter of alterations) {+ const result = completeResult(fixture);+ alter(result.findings[0]);+ const report = gradeResult(result, fixture);+ assert.equal(report.recall, 0.75);+ assert.equal(report.unmatchedFindings.length, 1);+ assert.equal(report.needsAdjudication, true);+ assert.equal(report.safeControlFindings, 0);+ assert.deepEqual(report.evidenceErrors, []);+ }+});++test("missing, fabricated, misplaced, or unrelated evidence cannot earn recall", () => {+ const fixture = createFixture();+ const alterations: Array<(finding: FixtureFinding) => void> = [+ (finding) => {+ finding.codeEvidence = [];+ },+ (finding) => {+ finding.codeEvidence[0].code = "invented source text";+ },+ (finding) => {+ finding.codeEvidence[0].startLine++;+ },+ (finding) => {+ finding.codeEvidence[0].path = "src/search.py";+ },+ (finding) => {+ finding.codeEvidence = [+ {+ path: "src/invoices.py",+ startLine: 1,+ code: fixture.files["src/invoices.py"].split("\n")[0],+ },+ ];+ },+ ];+ for (const alter of alterations) {+ const result = completeResult(fixture);+ alter(result.findings[0]);+ const report = gradeResult(result, fixture);+ assert.equal(report.recall, 0.75);+ assert.equal(report.passed, false);+ assert.ok(report.evidenceErrors.length > 0);+ }+});++test("control accusations and unmatched findings have separate adjudication queues", () => {+ const fixture = createFixture();+ const result = completeResult(fixture);+ result.findings.push(findingFor(fixture, fixture.controls[0]));+ const unmatched = findingFor(fixture, fixture.positives[1]);+ unmatched.taxonomy.cwe = ["CWE-400"];+ result.findings.push(unmatched);+ const report = gradeResult(result, fixture);+ assert.equal(report.recall, 1);+ assert.equal(report.safeControlFindings, 1);+ assert.deepEqual(report.safeControlDetails[0].controlIds, [+ fixture.controls[0].id,+ ]);+ assert.equal(report.unmatchedFindings.length, 1);+ assert.equal(report.passed, false);+ assert.equal(report.needsAdjudication, true);+});++test("safe-control context does not become a control accusation", () => {+ const fixture = createFixture();+ const result = completeResult(fixture);+ const context = findingFor(fixture, fixture.controls[0]);+ context.locations[0].role = "expected_control";+ result.findings[0].locations.push(...context.locations);+ result.findings[0].codeEvidence.push(...context.codeEvidence);+ assert.equal(gradeResult(result, fixture).passed, true);+});++test("descriptive safe-control roles require adjudication rather than a false-positive verdict", () => {+ const fixture = createFixture();+ const result = completeResult(fixture);+ const context = findingFor(fixture, fixture.controls[0]);+ context.locations[0].role =+ "safe sibling showing the expected tenant control";+ result.findings[0].locations.push(...context.locations);+ result.findings[0].codeEvidence.push(...context.codeEvidence);+ const report = gradeResult(result, fixture);+ assert.equal(report.recall, 1);+ assert.equal(report.safeControlFindings, 1);+ assert.equal(report.needsAdjudication, true);+ assert.equal(report.passed, false);+ assert.deepEqual(report.safeControlDetails[0].controlIds, [+ fixture.controls[0].id,+ ]);+});++test("source snippets may span the sink", () => {+ const fixture = createFixture();+ const result = completeResult(fixture);+ for (const finding of result.findings) {+ const evidence = finding.codeEvidence[0];+ const source = fixture.files[evidence.path];+ evidence.startLine--;+ evidence.code = source+ .split("\n")+ .slice(evidence.startLine - 1, evidence.startLine + 1)+ .join("\n");+ }+ assert.equal(gradeResult(result, fixture).passed, true);+});++test("fixture source is deterministic and contains no gold identifiers", () => {+ const fixture = createFixture();+ assert.deepEqual(createFixture(), fixture);+ const source = Object.values(fixture.files).join("\n");+ for (const expected of [...fixture.positives, ...fixture.controls]) {+ assert.equal(source.includes(expected.id), false);+ }+});diff --git a/evals/standard-time-budget/test_production.mts b/evals/standard-time-budget/test_production.mtsnew file mode 100644index 000000000..22ad5d236--- /dev/null+++ b/evals/standard-time-budget/test_production.mts@@ -0,0 +1,378 @@+import assert from "node:assert/strict";+import { existsSync, promises as fs } from "node:fs";+import { cp, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";+import { syncBuiltinESMExports } from "node:module";+import { join, relative } from "node:path";+import test from "node:test";+import { fileURLToPath } from "node:url";+import { temporaryDirectory } from "../../plugins/codex-security/mcp-app/tests/support/temporary-directories.ts";+import type { ThreadEvent } from "../../sdk/typescript/node_modules/@openai/codex-sdk/dist/index.js";+import type { JsonObject } from "../../sdk/typescript/src/config.js";+import * as sdk from "../../sdk/typescript/dist/api.js";+import { createIsolatedHome } from "../../sdk/typescript/dist/runtime.js";+import { withEvalState } from "../secret-discovery/runtime.mts";+import { hashText, runComparison } from "./experiment.mts";+import {+ createStandardRunner,+ preflightStandardEval,+ snapshotCodexExecutable,+} from "./production.mts";+import { prepareStandardEval } from "./fixtures.mts";+import { summarizeRuns } from "./summary.mts";++const { CodexSecurity } =+ sdk as unknown as typeof import("../../sdk/typescript/src/api.js");++test("executable provenance follows the configured binary and its actual bytes", async (context) => {+ const root = await temporaryDirectory("time-budget-executable-");+ context.after(() => rm(root, { recursive: true, force: true }));+ const first = join(root, "first.exe");+ const second = join(root, "second.exe");+ await writeFile(first, "synthetic executable one");+ await writeFile(second, "synthetic executable two");+ const snapshot = await snapshotCodexExecutable({ CODEX_CLI_PATH: first });+ assert.deepEqual(snapshot, {+ command: first,+ files: [+ {+ path: await realpath(first),+ sha256: hashText("synthetic executable one"),+ },+ ],+ });+ assert.deepEqual(await snapshotCodexExecutable({ CODEX_CLI_PATH: second }), {+ command: second,+ files: [+ {+ path: await realpath(second),+ sha256: hashText("synthetic executable two"),+ },+ ],+ });+ await writeFile(first, "replaced executable");+ const replaced = await snapshotCodexExecutable({ CODEX_CLI_PATH: first });+ assert.equal(replaced.command, snapshot.command);+ assert.equal(replaced.files[0]!.sha256, hashText("replaced executable"));+ assert.notEqual(replaced.files[0]!.sha256, snapshot.files[0]!.sha256);+});++test("state cleanup failure saves the attempt and stops before the paired arm", async (context) => {+ const root = await temporaryDirectory("time-budget-cleanup-");+ const originalRm = fs.rm;+ context.after(async () => {+ context.mock.restoreAll();+ syncBuiltinESMExports();+ await originalRm(root, { recursive: true, force: true });+ });+ const prepared = await prepareStandardEval(root, "/synthetic/plugin");+ const outputRoot = join(root, "results");+ const states: string[] = [];+ const saved: Awaited<ReturnType<typeof runComparison>> = [];+ context.mock.method(fs, "rm", async (...args: Parameters<typeof fs.rm>) => {+ if (states.includes(String(args[0]))) {+ throw new Error("Synthetic state removal failure");+ }+ return originalRm(...args);+ });+ syncBuiltinESMExports();+ await assert.rejects(+ runComparison({+ prepared,+ config: { model: "eval-model", pairs: 2, budgetMinutes: 5 },+ run: createStandardRunner({+ stateRoot: join(root, "states"),+ outputRoot,+ createClient: (_config, dependencies) => {+ states.push(dependencies.environment["CODEX_SECURITY_STATE_DIR"]!);+ return {+ run: async (_target, options) => {+ await writeFile(join(options!.outputDir!, "partial.json"), "{}");+ throw new Error("Synthetic scan failure");+ },+ close: async () => {+ throw new Error("Synthetic client close failure");+ },+ };+ },+ }),+ onResult: async (record) => {+ saved.push(record);+ },+ }),+ /Synthetic state removal failure/,+ );+ assert.equal(states.length, 1);+ assert.equal(saved.length, 1);+ assert.equal(saved[0]!.status, "failed");+ assert.match(saved[0]!.error!, /Synthetic scan failure/);+ assert.match(saved[0]!.error!, /Synthetic client close failure/);+ assert.match(saved[0]!.error!, /Synthetic state removal failure/);+ assert.equal(existsSync(states[0]!), true);+ assert.equal(+ await readFile(+ join(outputRoot, "pair-1-baseline", "scan", "partial.json"),+ "utf8",+ ),+ "{}",+ );+});++test("the developer eval preflights API-key authentication and the production budget before staging", async () => {+ assert.throws(+ () => preflightStandardEval({}, 5),+ /requires OPENAI_API_KEY or CODEX_API_KEY/,+ );+ for (const name of ["OPENAI_API_KEY", "CODEX_API_KEY"]) {+ preflightStandardEval({ [name]: "synthetic-environment" }, 0.25);+ }+ assert.throws(+ () =>+ preflightStandardEval({ OPENAI_API_KEY: "synthetic-environment" }, 1e12),+ /deadline within the supported date range/,+ );+ await withEvalState(createIsolatedHome, undefined, async ({ home }) => {+ assert.equal(existsSync(join(home, "auth.json")), false);+ });+});++async function writeUsageSession(+ home: string,+ threadId: string,+ usage: Record<string, number>,+ parent?: string,+) {+ const directory = join(home, "sessions", "2026", "01", "01");+ await mkdir(directory, { recursive: true });+ await writeFile(+ join(directory, `rollout-${threadId}.jsonl`),+ [+ JSON.stringify({+ type: "session_meta",+ payload: {+ id: threadId,+ ...(parent === undefined ? {} : { parent_thread_id: parent }),+ },+ }),+ JSON.stringify({+ type: "event_msg",+ payload: {+ type: "token_count",+ info: { total_token_usage: usage },+ },+ }),+ "",+ ].join("\n"),+ );+}++test("the paired evaluator preserves aggregate usage across production completion and failure", async (context) => {+ const root = await temporaryDirectory("time-budget-production-");+ context.after(() => rm(root, { recursive: true, force: true }));+ const pluginRoot = fileURLToPath(+ new URL("../../plugins/codex-security/", import.meta.url),+ );+ const prepared = await prepareStandardEval(root, pluginRoot);+ const outputRoot = join(root, "results");+ await mkdir(outputRoot);+ await writeFile(+ join(outputRoot, "fixture.json"),+ JSON.stringify(prepared.fixture),+ );+ const states: string[] = [];+ const commands: string[] = [];+ const prompts: string[] = [];+ const records = await runComparison({+ prepared,+ config: { model: "gpt-5.6-sol", pairs: 2, budgetMinutes: 5 },+ run: createStandardRunner({+ stateRoot: join(root, "states"),+ outputRoot,+ environment: { OPENAI_API_KEY: "synthetic-environment" },+ createClient: (config, dependencies) => {+ const home = dependencies.environment["CODEX_SECURITY_STATE_DIR"]!;+ states.push(home);+ const client = new CodexSecurity(+ config,+ {+ ...dependencies,+ now: () => Date.UTC(2026, 0, 1),+ probeCodexSandbox: async () => {},+ prepareScanArtifactRestorer: async () => ({+ restore: async () => {},+ }),+ prepareRuntime: async () => ({+ codexHome: home,+ plugin: {+ pluginRoot,+ marketplaceRoot: pluginRoot,+ installedRoot: pluginRoot,+ marketplaceName: "codex-security-sdk",+ name: "codex-security",+ version: "0.1.0",+ },+ environment: {},+ credentialsAvailable: true,+ }),+ resolvePluginPython: async () => "/managed/python",+ repositoryRevision: async () => "deadbeef",+ runWorkbench: async (_options, args): Promise<JsonObject> => {+ commands.push(args[0]!);+ if (+ args[0] === "prepare-scan-completion" &&+ states.length === 1+ ) {+ throw new Error(+ "Synthetic completion failure after model usage",+ );+ }+ if (args[0] === "register-cli-scan") {+ return {+ scanId: "scan_example_001",+ targetId: "target_sha256_example",+ targetRevision: "deadbeef",+ scanDir: args[args.indexOf("--scan-dir") + 1]!,+ contract: { target: { allowedKinds: ["git_revision"] } },+ };+ }+ if (args[0] === "get-scan-feedback") {+ return {+ scanId: "scan_example_001",+ targetId: "target_sha256_example",+ falsePositives: [],+ };+ }+ return {};+ },+ },+ { surface: "sdk" },+ );+ return {+ run: client.run.bind(client),+ close: async () => {+ await client.close();+ if (states.length <= 2)+ throw new Error("Synthetic cleanup failure");+ },+ };+ },+ createCodex: async (options) => ({+ startThread: (threadOptions) => ({+ id: "thread-1",+ runStreamed: async (prompt) => {+ prompts.push(prompt);+ const scanDir = options.env!["CODEX_SECURITY_SCAN_DIR"]!;+ const home = options.env!["CODEX_HOME"]!;+ const nearby = join(scanDir, "..", "..");+ assert.equal(threadOptions?.workingDirectory, scanDir);+ for (const path of [+ scanDir,+ home,+ options.env!["CODEX_SECURITY_STATE_DIR"]!,+ ]) {+ assert.doesNotMatch(relative(root, path), /baseline|time-aware/u);+ }+ assert.equal(existsSync(join(nearby, "fixture.json")), false);+ assert.equal(+ existsSync(join(nearby, "pair-1-time-aware", "result.json")),+ false,+ );+ for (const previous of states.slice(0, -1)) {+ assert.equal(existsSync(previous), false);+ }+ await writeUsageSession(home, "thread-1", {+ input_tokens: 100,+ output_tokens: 20,+ });+ await writeUsageSession(+ home,+ "worker-thread",+ { input_tokens: 50, output_tokens: 7 },+ "thread-1",+ );+ await cp(join(pluginRoot, "examples", "completed-scan"), scanDir, {+ recursive: true,+ });+ await writeFile(+ join(scanDir, "report.md"),+ "# Finalized synthetic report\n",+ );+ async function* events(): AsyncGenerator<ThreadEvent> {+ yield { type: "thread.started", thread_id: "thread-1" };+ yield { type: "turn.started" };+ yield {+ type: "turn.completed",+ usage: {+ input_tokens: 10,+ cached_input_tokens: 2,+ cache_write_input_tokens: 0,+ output_tokens: 3,+ reasoning_output_tokens: 1,+ },+ };+ }+ return { events: events() };+ },+ }),+ }),+ }),+ onResult: async (record, result) => {+ assert.equal(existsSync(states.at(-1)!), false);+ if (result !== undefined) {+ await writeFile(+ join(+ outputRoot,+ `pair-${record.pair}-${record.variant}`,+ "result.json",+ ),+ JSON.stringify(result),+ );+ }+ },+ });+ assert.equal(new Set(states).size, 4);+ assert.equal(prompts.length, 4, JSON.stringify(records));+ assert.ok(!prompts[0].includes("# Investigation time allowance"));+ assert.ok(prompts[1].includes("2026-01-01T00:05:00.000Z"));+ assert.deepEqual(+ records.map((record) => record.status),+ ["failed", "failed", "completed", "completed"],+ JSON.stringify(records),+ );+ assert.match(+ records[0].error!,+ /Synthetic completion failure after model usage/,+ );+ assert.match(records[0].error!, /Synthetic cleanup failure/);+ assert.equal(records[1].error, "Synthetic cleanup failure");+ assert.equal(+ commands.filter((command) => command === "prepare-scan-completion").length,+ 4,+ );+ assert.equal(+ commands.filter((command) => command === "complete-scan").length,+ 3,+ );+ for (const [index, record] of records.entries()) {+ assert.partialDeepStrictEqual(record.usage, {+ input_tokens: 150,+ output_tokens: 27,+ });+ const archivedReport = join(+ outputRoot,+ `pair-${record.pair}-${record.variant}`,+ "scan",+ "report.md",+ );+ assert.match(+ await readFile(archivedReport, "utf8"),+ /Finalized synthetic report/,+ );+ if (record.status === "completed")+ assert.equal(record.report.reportPath, archivedReport);+ assert.equal(record.promptSha256, hashText(prompts[index]));+ }+ const summary = summarizeRuns(records);+ assert.equal(summary.arms.baseline.usage.total_tokens, 354);+ assert.equal(summary.paired.completedPairs, 1);+ assert.equal(summary.paired.excludedPairs, 1);+});diff --git a/evals/standard-time-budget/test_summary.mts b/evals/standard-time-budget/test_summary.mtsnew file mode 100644index 000000000..8ea3678d5--- /dev/null+++ b/evals/standard-time-budget/test_summary.mts@@ -0,0 +1,189 @@+import assert from "node:assert/strict";+import test from "node:test";+import { summarizeRuns } from "./summary.mts";+import { createFixture } from "./fixtures.mts";+import { gradeResult } from "./grade.mts";+import type { CompletedRunRecord, RunReport, Variant } from "./experiment.mts";++function attempt(pair: number, variant: Variant) {+ return {+ pair,+ variant,+ elapsedMs: 100,+ startedAt: "2026-01-01T00:00:00.000Z",+ advisoryBudgetMinutes: variant === "time-aware" ? 5 : null,+ promptSha256: null,+ usage: null,+ };+}++function completed(+ pair: number,+ variant: Variant,+ overrides: Partial<RunReport> = {},+): CompletedRunRecord {+ return {+ ...attempt(pair, variant),+ status: "completed",+ usage: { input_tokens: 100, cached_input_tokens: 40, output_tokens: 20 },+ report: {+ ...gradeResult({ findings: [] }, createFixture()),+ recall: 0.5,+ foundCount: 1,+ expectedCount: 2,+ safeControlFindings: 0,+ unmatchedFindings: [],+ evidenceErrors: [],+ ...overrides,+ },+ };+}++test("pairs runs by identifier and reports time-aware minus baseline", () => {+ const result = summarizeRuns([+ completed(2, "time-aware", { recall: 1, foundCount: 2 }),+ completed(1, "baseline"),+ completed(2, "baseline"),+ {+ ...completed(1, "time-aware", {+ recall: 0,+ foundCount: 0,+ safeControlFindings: 1,+ unmatchedFindings: [+ { findingIndex: 0, title: "unmatched", reason: "unknown source" },+ ],+ evidenceErrors: [+ { reason: "bad source" },+ { reason: "missing evidence" },+ ],+ }),+ elapsedMs: 300,+ },+ ]);+ assert.equal(result.arms.baseline.attempted, 2);+ assert.equal(result.arms.baseline.successRate, 1);+ assert.equal(result.arms["time-aware"].meanElapsedMs, 200);+ assert.equal(result.arms["time-aware"].meanRecall, 0.5);+ assert.equal(result.arms["time-aware"].foundCount, 2);+ assert.equal(result.arms["time-aware"].expectedCount, 4);+ assert.equal(result.arms["time-aware"].safeControlFindings, 1);+ assert.equal(result.arms["time-aware"].unmatchedFindings, 1);+ assert.equal(result.arms["time-aware"].evidenceErrors, 2);+ assert.equal(result.paired.completedPairs, 2);+ assert.equal(result.paired.excludedPairs, 0);+ assert.equal(result.paired.meanDelta.recall, 0);+ assert.equal(result.paired.meanDelta.elapsedMs, 100);+ assert.equal(result.paired.meanDelta.safeControlFindings, 0.5);+ assert.equal(result.paired.meanDelta.evidenceErrors, 1);+});++test("shows failures and missing arms alongside completed-pair improvements", () => {+ const result = summarizeRuns([+ completed(1, "baseline"),+ completed(1, "time-aware", { recall: 1, foundCount: 2 }),+ completed(2, "baseline"),+ {+ ...attempt(2, "time-aware"),+ status: "failed",+ error: "scan failed",+ elapsedMs: 900,+ },+ completed(3, "baseline"),+ ]);+ assert.equal(result.arms["time-aware"].attempted, 2);+ assert.equal(result.arms["time-aware"].completed, 1);+ assert.equal(result.arms["time-aware"].failed, 1);+ assert.equal(result.arms["time-aware"].successRate, 0.5);+ assert.equal(result.arms["time-aware"].meanElapsedMs, 500);+ assert.equal(result.arms["time-aware"].meanRecall, 1);+ assert.equal(result.paired.totalPairs, 3);+ assert.equal(result.paired.completedPairs, 1);+ assert.equal(result.paired.excludedPairs, 2);+ assert.equal(result.paired.meanDelta.recall, 0.5);+ assert.equal(result.arms["time-aware"].usage.input_tokens, null);+ assert.equal(result.arms["time-aware"].usage.total_tokens, null);+ assert.equal(result.arms["time-aware"].usage.missingUsageRuns, 1);+});++test("cached input tokens are a subset and never added to total tokens", () => {+ const result = summarizeRuns([+ completed(1, "baseline"),+ {+ ...completed(1, "time-aware"),+ usage: { input_tokens: 150, cached_input_tokens: 80, output_tokens: 40 },+ },+ ]);+ assert.deepEqual(result.arms.baseline.usage, {+ input_tokens: 100,+ cached_input_tokens: 40,+ output_tokens: 20,+ total_tokens: 120,+ missingUsageRuns: 0,+ });+ assert.equal(result.paired.meanDelta.input_tokens, 50);+ assert.equal(result.paired.meanDelta.cached_input_tokens, 40);+ assert.equal(result.paired.meanDelta.output_tokens, 20);+ assert.equal(result.paired.meanDelta.total_tokens, 70);+});++test("missing usage remains unknown without discarding completed pairs", () => {+ const result = summarizeRuns([+ completed(1, "baseline"),+ { ...completed(1, "time-aware"), usage: null },+ completed(2, "baseline"),+ completed(2, "time-aware"),+ ]);+ assert.equal(result.arms["time-aware"].usage.input_tokens, null);+ assert.equal(result.arms["time-aware"].usage.cached_input_tokens, null);+ assert.equal(result.arms["time-aware"].usage.output_tokens, null);+ assert.equal(result.arms["time-aware"].usage.total_tokens, null);+ assert.equal(result.arms["time-aware"].usage.missingUsageRuns, 1);+ assert.equal(result.paired.completedPairs, 2);+ assert.equal(result.paired.meanDelta.total_tokens, null);+ assert.equal(result.paired.meanDelta.recall, 0);+});++test("a missing cache count does not hide known input and output totals", () => {+ const result = summarizeRuns([+ {+ ...completed(1, "baseline"),+ usage: { input_tokens: 100, output_tokens: 20 },+ },+ ]);+ assert.equal(result.arms.baseline.usage.input_tokens, 100);+ assert.equal(result.arms.baseline.usage.cached_input_tokens, null);+ assert.equal(result.arms.baseline.usage.total_tokens, 120);+ assert.equal(result.arms.baseline.usage.missingUsageRuns, 1);+});++test("empty and entirely failed arms have no measured quality or deltas", () => {+ const empty = summarizeRuns([]);+ assert.equal(empty.arms.baseline.attempted, 0);+ assert.equal(empty.arms.baseline.successRate, null);+ assert.equal(empty.arms.baseline.meanElapsedMs, null);+ assert.equal(empty.arms.baseline.usage.total_tokens, null);+ assert.equal(empty.paired.totalPairs, 0);+ assert.equal(empty.paired.meanDelta.recall, null);+ const failed = summarizeRuns([+ { ...attempt(1, "baseline"), status: "failed", error: "scan failed" },+ {+ ...attempt(1, "time-aware"),+ status: "failed",+ error: "scan failed",+ elapsedMs: 200,+ },+ ]);+ assert.equal(failed.arms.baseline.successRate, 0);+ assert.equal(failed.arms.baseline.meanRecall, null);+ assert.equal(failed.arms.baseline.safeControlFindings, null);+ assert.equal(failed.paired.completedPairs, 0);+ assert.equal(failed.paired.excludedPairs, 1);+ assert.equal(failed.paired.meanDelta.elapsedMs, null);+});++test("duplicate arms cannot silently bias results", () => {+ assert.throws(+ () => summarizeRuns([completed(1, "baseline"), completed(1, "baseline")]),+ /Duplicate run for pair 1\/baseline/,+ );+});diff --git a/evals/tsconfig.json b/evals/tsconfig.jsonindex 354d2a787..18228790b 100644--- a/evals/tsconfig.json+++ b/evals/tsconfig.json@@ -1,12 +1,14 @@{"extends": "../plugins/codex-security/mcp-app/tsconfig.json","compilerOptions": {+ "lib": ["esnext", "dom", "dom.iterable"],"typeRoots": ["../sdk/typescript/node_modules/@types"]},"include": ["../plugins/codex-security/mcp-app/src/assets.d.ts","deep-reducer/*.mts","secret-discovery/*.mts",+ "standard-time-budget/*.mts","workflow-routing/*.mts","triage-finding/assertions/*.mts","triage-finding/scripts/*.mts",diff --git a/sdk/typescript/package.json b/sdk/typescript/package.jsonindex 28f773f5f..b59fec017 100644--- a/sdk/typescript/package.json+++ b/sdk/typescript/package.json@@ -66,7 +66,7 @@"build:plugin": "node scripts/build-plugin.mjs","check:plugin-source": "node scripts/check-plugin-source.mjs","check:package": "node scripts/check-package.mjs",- "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.mts\" \"../../plugins/codex-security/native/*.mts\" \"../../evals/{deep-reducer,secret-discovery,workflow-routing}/*.mts\" \"../../evals/triage-finding/{assertions,scripts,sastbench/assertions,sastbench/scripts}/*.mts\" \"../../evals/triage-finding/types.ts\" \"../../evals/tsconfig.json\" && node scripts/check-markdown.mjs",+ "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.mts\" \"../../plugins/codex-security/native/*.mts\" \"../../evals/{deep-reducer,secret-discovery,standard-time-budget,workflow-routing}/*.mts\" \"../../evals/triage-finding/{assertions,scripts,sastbench/assertions,sastbench/scripts}/*.mts\" \"../../evals/triage-finding/types.ts\" \"../../evals/tsconfig.json\" && node scripts/check-markdown.mjs","generate:models": "node scripts/generate-models.cjs","generate:models:check": "node scripts/generate-models.cjs --check","generate:external-import-models": "node scripts/generate-external-import-models.cjs",diff --git a/sdk/typescript/tests-ts/scan-directory-identity.test.ts b/sdk/typescript/tests-ts/scan-directory-identity.test.tsnew file mode 100644index 000000000..3fc882ad3--- /dev/null+++ b/sdk/typescript/tests-ts/scan-directory-identity.test.ts@@ -0,0 +1,214 @@+import { afterEach, expect, spyOn, test } from "bun:test";+import {+ chmod,+ cp,+ mkdir,+ readFile,+ rename,+ rm,+ writeFile,+ type FileHandle,+} from "node:fs/promises";+import type { BigIntStats, Stats } from "node:fs";+import { join } from "node:path";+import { loadContract, readScanFile } from "../src/contract.js";+import { ContractValidationError } from "../src/errors.js";+import { PLUGIN_ROOT } from "./plugin-root.js";+import { createApiTestFixtures } from "./support/temporary-directories.js";+import { runTestInSubprocess } from "./support/test-subprocess.js";++const { temporaryDirectory, cleanup } = createApiTestFixtures(+ "codex-security-directory-identity-",+);+afterEach(cleanup);+const scenarios = [+ "initial-root",+ "accepted-root",+ "between-documents",+ "parent-before-leaf",+ "parent-retains-leaf",+] as const;+const identities = [+ ["small-distinct", 100n, 101n],+ ["rounded-distinct", 2n ** 60n, 2n ** 60n + 1n],+ ["unchanged", 2n ** 60n, 2n ** 60n],+] as const;++for (const scenario of scenarios) {+ for (const [kind, originalId, replacementId] of identities) {+ const title = `checked scan directory ${scenario} preserves ${kind} identities`;+ test(title, async () => {+ if (runTestInSubprocess(import.meta.path, title)) return;+ const root = await temporaryDirectory();+ const scanDir = join(root, "scan");+ const isParent = scenario.startsWith("parent-");+ const swappedPath = isParent ? join(scanDir, "artifacts") : scanDir;+ const replacementPath = join(root, "replacement");+ const backupPath = join(root, "original");+ const leaf = join(swappedPath, "document.json");+ if (isParent) {+ await mkdir(swappedPath, { recursive: true, mode: 0o700 });+ await chmod(scanDir, 0o700);+ await mkdir(replacementPath, { mode: 0o700 });+ await writeFile(leaf, "original data");+ await writeFile(+ join(replacementPath, "document.json"),+ "replacement data",+ );+ } else {+ await cp(join(PLUGIN_ROOT, "examples", "completed-scan"), scanDir, {+ recursive: true,+ });+ await cp(+ join(PLUGIN_ROOT, "examples", "completed-scan"),+ replacementPath,+ { recursive: true },+ );+ await chmod(scanDir, 0o700);+ await chmod(replacementPath, 0o700);+ for (const [directory, name] of [+ [scanDir, "original contract"],+ [replacementPath, "replacement contract"],+ ]) {+ const path = join(directory!, "scan-manifest.json");+ const manifest = JSON.parse(await readFile(path, "utf8"));+ manifest.scan.target.displayName = name;+ await writeFile(path, JSON.stringify(manifest));+ }+ }+ const files = await import("node:fs/promises");+ const originalLstat = files.lstat;+ const originalRealpath = files.realpath;+ const originalOpen = files.open;+ const runtime = await import("../src/runtime.js");+ const originalAncestry = runtime.requireSecureOutputAncestry;+ const originalDirectory = await originalLstat(swappedPath, {+ bigint: true,+ });+ const replacementDirectory = await originalLstat(replacementPath, {+ bigint: true,+ });+ expect(originalDirectory.ino).not.toBe(replacementDirectory.ino);+ const originalLeaf = isParent+ ? await originalLstat(leaf, { bigint: true })+ : undefined;+ let swapped = false;+ const handles: FileHandle[] = [];+ const restoreCloses: Array<() => void> = [];+ const swap = async () => {+ if (swapped || kind === "unchanged") return;+ await rename(swappedPath, backupPath);+ await rename(replacementPath, swappedPath);+ if (scenario === "parent-retains-leaf") {+ await rm(leaf);+ await rename(join(backupPath, "document.json"), leaf);+ }+ swapped = true;+ };+ const withIdentity = (metadata: Stats | BigIntStats) => {+ Object.defineProperty(metadata, "ino", {+ value:+ typeof metadata.ino === "bigint"+ ? swapped+ ? replacementId+ : originalId+ : Number(swapped ? replacementId : originalId),+ });+ return metadata;+ };+ const inspecting = spyOn(files, "lstat").mockImplementation((async (+ ...args: Parameters<typeof originalLstat>+ ) => {+ if (scenario === "parent-before-leaf" && args[0] === leaf) await swap();+ const metadata = await originalLstat(...args);+ return args[0] === swappedPath ? withIdentity(metadata) : metadata;+ }) as typeof originalLstat);+ const resolving = spyOn(files, "realpath").mockImplementation((async (+ ...args: Parameters<typeof originalRealpath>+ ) => {+ const canonical = await originalRealpath(...args);+ if (+ (scenario === "initial-root" && args[0] === scanDir) ||+ (scenario === "parent-retains-leaf" && args[0] === leaf)+ )+ await swap();+ return canonical;+ }) as typeof originalRealpath);+ const ancestry = spyOn(+ runtime,+ "requireSecureOutputAncestry",+ ).mockImplementation(async (...args) => {+ await originalAncestry(...args);+ if (scenario === "accepted-root" && args[0] === scanDir) await swap();+ });+ const opening = spyOn(files, "open").mockImplementation(+ async (...args) => {+ const handle = await originalOpen(...args);+ handles.push(handle);+ if (+ scenario === "between-documents" &&+ args[0] === join(scanDir, "scan-manifest.json")+ ) {+ const close = handle.close.bind(handle);+ const closing = spyOn(handle, "close").mockImplementation(+ async () => {+ await close();+ await swap();+ },+ );+ restoreCloses.push(() => closing.mockRestore());+ }+ return handle;+ },+ );+ let value: string | undefined;+ let error: unknown;+ try {+ try {+ value = isParent+ ? (+ await readScanFile(+ scanDir,+ "artifacts/document.json",+ "fixture",+ )+ ).toString()+ : (+ await loadContract(scanDir, {+ pluginRoot: PLUGIN_ROOT,+ expectedScanId: "scan_example_001",+ })+ ).manifest.scan.target.displayName;+ } catch (caught) {+ error = caught;+ }+ const finalDirectory = await originalLstat(swappedPath, {+ bigint: true,+ });+ const finalLeaf = isParent+ ? await originalLstat(leaf, { bigint: true })+ : undefined;+ expect(swapped).toBe(kind !== "unchanged");+ for (const handle of handles) expect(handle.fd).toBe(-1);+ if (isParent || scenario === "between-documents")+ expect(handles.length).toBeGreaterThan(0);+ if (kind === "unchanged") {+ expect(finalDirectory.ino).toBe(originalDirectory.ino);+ expect(error).toBeUndefined();+ expect(value).toBe(isParent ? "original data" : "original contract");+ } else {+ expect(finalDirectory.ino).toBe(replacementDirectory.ino);+ expect(error).toBeInstanceOf(ContractValidationError);+ }+ if (scenario === "parent-retains-leaf")+ expect(finalLeaf?.ino).toBe(originalLeaf?.ino);+ } finally {+ inspecting.mockRestore();+ resolving.mockRestore();+ opening.mockRestore();+ ancestry.mockRestore();+ for (const restore of restoreCloses) restore();+ }+ });+ }+}