Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(tests): simplify fixtures and eval tooling (#1292)

mldangelo-oai3950fe634150
diff --git a/evals/triage-finding/sastbench/assertions/sastbench-metrics.mts b/evals/triage-finding/sastbench/assertions/sastbench-metrics.mts
index dab7ec7ac..0542d7140 100644
--- a/evals/triage-finding/sastbench/assertions/sastbench-metrics.mts
+++ b/evals/triage-finding/sastbench/assertions/sastbench-metrics.mts
@@ -16,8 +16,7 @@ export function afterEach(context: {
test: PromptfooTest;
}) {
const row = context.result;
- const vars = row.vars || context.test.vars;
- const caseId = String(vars?.case_id || "");
+ const caseId = String(row.vars.case_id || "");
const expectedGroundTruth = String(context.test.metadata?.ground_truth || "");
const output = row.response?.output;
let status = "model_error";
diff --git a/evals/triage-finding/sastbench/scripts/sastbench-lib.mts b/evals/triage-finding/sastbench/scripts/sastbench-lib.mts
index e76b9a9b5..d984191df 100644
--- a/evals/triage-finding/sastbench/scripts/sastbench-lib.mts
+++ b/evals/triage-finding/sastbench/scripts/sastbench-lib.mts
@@ -201,13 +201,6 @@ export function validateDataset(
return { caseCount: records.length, labelCounts: counts };
}
-export function opaqueCaseId(index: number) {
- if (!Number.isInteger(index) || index < 0) {
- throw new Error("SastBench case index must be a non-negative integer");
- }
- return `sastbench-${String(index).padStart(6, "0")}`;
-}
-
export function repositoryStateId(
record: Pick<SastBenchRecord, "repo_url" | "commit_hash">,
) {
@@ -250,7 +243,7 @@ export function buildPromptVars(
index: number,
targetRoot: string | undefined = DEFAULT_TARGET_ROOT,
) {
- const caseId = opaqueCaseId(index);
+ const caseId = `sastbench-${String(index).padStart(6, "0")}`;
return {
case_id: caseId,
target_repo: path.join(path.resolve(targetRoot), repositoryStateId(record)),
diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
index a5de447a7..c670bed32 100644
--- a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
@@ -10,7 +10,6 @@ import {
generateSampleTests,
generatePromptfooTests,
generateRepresentativeSampleTests,
- opaqueCaseId,
repositoryStateId,
selectRepresentativeSample,
validateDataset,
@@ -101,9 +100,10 @@ assert.doesNotMatch(sparsePromptVars.finding_input, /claim:\s*\n/);
assert.match(sparsePromptVars.finding_input, /src\/file-level\.js:0-100/);
assert.match(sparsePromptVars.finding_input, /src\/zero-width\.js:54-53/);
-assert.equal(opaqueCaseId(0), "sastbench-000000");
-assert.equal(opaqueCaseId(2736), "sastbench-002736");
-assert.throws(() => opaqueCaseId(-1), /non-negative integer/);
+assert.equal(
+ buildPromptVars(positive, 2736, targetRoot).case_id,
+ "sastbench-002736",
+);
const stateId = repositoryStateId(positive);
assert.match(stateId, /^state-[a-f0-9]{16}$/);
diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts
index 40a53598e..73aa2e259 100644
--- a/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts
+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-result.mts
@@ -92,13 +92,13 @@ function extensionContext({
}) {
return {
test: {
- vars: { case_id: caseId },
metadata: {
ground_truth: expectedGroundTruth,
...(sampleWeight === undefined ? {} : { sample_weight: sampleWeight }),
},
},
result: {
+ vars: { case_id: caseId },
latencyMs: 1234,
cost: 0.25,
namedScores: { existing_metric: 1 },
diff --git a/evals/triage-finding/types.ts b/evals/triage-finding/types.ts
index c5ed7a0a7..4719f10aa 100644
--- a/evals/triage-finding/types.ts
+++ b/evals/triage-finding/types.ts
@@ -4,12 +4,11 @@ export interface AssertionContext {
}
export interface PromptfooTest {
- vars?: Record<string, unknown>;
metadata?: Record<string, unknown>;
}
export interface PromptfooRow {
- vars?: Record<string, unknown>;
+ vars: Record<string, unknown>;
latencyMs?: number;
cost?: number;
response?: {
diff --git a/sdk/typescript/scripts/test-shards.mts b/sdk/typescript/scripts/test-shards.mts
index 015458e26..bf6db5906 100644
--- a/sdk/typescript/scripts/test-shards.mts
+++ b/sdk/typescript/scripts/test-shards.mts
@@ -4,9 +4,6 @@ export function shardTestFiles(
count: number,
durations: Readonly<Record<string, number>> = {},
) {
- if (!Number.isSafeInteger(count) || count < 1) {
- throw new Error("The test shard count must be a positive integer.");
- }
const shards = Array.from({ length: count }, () => ({
files: [] as string[],
seconds: 0,
diff --git a/sdk/typescript/tests-ts/api-credentials.test.ts b/sdk/typescript/tests-ts/api-credentials.test.ts
index 105bd5f89..b37708141 100644
--- a/sdk/typescript/tests-ts/api-credentials.test.ts
+++ b/sdk/typescript/tests-ts/api-credentials.test.ts
@@ -9,14 +9,13 @@ import { afterEach, describe, expect, test, mock } from "bun:test";
import { parse as parseToml } from "smol-toml";
import { initialCredentialsAvailable } from "../src/api.js";
import { setCodexSecurityCredentialLogout } from "../src/runtime.js";
-import { PLUGIN_ROOT } from "./plugin-root.js";
+import { copyCompletedScan, PLUGIN_ROOT } from "./plugin-root.js";
import { shellEnvironmentReference, TestClient } from "./support/api-client.js";
import { completedEvents, preparedRuntime } from "./support/api-events.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { rejecting } from "./support/errors.js";
-const { cleanup, copyCompletedScan, temporaryDirectory } =
- createApiTestFixtures();
+const { cleanup, temporaryDirectory } = createApiTestFixtures();
afterEach(cleanup);
describe("CodexSecurity orchestration", () => {
diff --git a/sdk/typescript/tests-ts/api-events.test.ts b/sdk/typescript/tests-ts/api-events.test.ts
index d59f8ecd3..d3bc1c220 100644
--- a/sdk/typescript/tests-ts/api-events.test.ts
+++ b/sdk/typescript/tests-ts/api-events.test.ts
@@ -1,3 +1,4 @@
+import { copyCompletedScan } from "./plugin-root.js";
import { once } from "node:events";
import { mkdir, stat } from "node:fs/promises";
import { existsSync } from "node:fs";
@@ -28,8 +29,7 @@ import {
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { throwing } from "./support/errors.js";
-const { cleanup, copyCompletedScan, temporaryDirectory } =
- createApiTestFixtures();
+const { cleanup, temporaryDirectory } = createApiTestFixtures();
afterEach(cleanup);
diff --git a/sdk/typescript/tests-ts/api-post-scan.test.ts b/sdk/typescript/tests-ts/api-post-scan.test.ts
index 809559ab5..36a6d4665 100644
--- a/sdk/typescript/tests-ts/api-post-scan.test.ts
+++ b/sdk/typescript/tests-ts/api-post-scan.test.ts
@@ -21,7 +21,7 @@ import {
} from "../src/runtime.js";
import * as runtime from "../src/runtime.js";
import { writeThreatModel } from "../src/artifact-export.js";
-import { PLUGIN_ROOT } from "./plugin-root.js";
+import { copyCompletedScan, PLUGIN_ROOT } from "./plugin-root.js";
import { TestClient } from "./support/api-client.js";
import {
completedEvents,
@@ -30,8 +30,7 @@ import {
} from "./support/api-events.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
-const { cleanup, copyCompletedScan, temporaryDirectory } =
- createApiTestFixtures();
+const { cleanup, temporaryDirectory } = createApiTestFixtures();
afterEach(cleanup);
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index 717d4735a..2562b5323 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -61,7 +61,11 @@ import {
import * as runtime from "../src/runtime.js";
import { matchScanFindingsInternal } from "../src/scan-comparison.js";
import { normalizeTarget } from "../src/targets.js";
-import { INTEGRATION_TARGET, PLUGIN_ROOT } from "./plugin-root.js";
+import {
+ copyCompletedScan,
+ INTEGRATION_TARGET,
+ PLUGIN_ROOT,
+} from "./plugin-root.js";
import {
mockScanRegistration,
mockWorkbench,
@@ -91,8 +95,7 @@ import { mockFs, restoreFs } from "./support/module-mocks.js";
const REPOSITORY_ROOT = fileURLToPath(new URL("../../..", import.meta.url));
const EXAMPLE = join(PLUGIN_ROOT, "examples", "completed-scan");
-const { cleanup, copyCompletedScan, temporaryDirectory } =
- createApiTestFixtures();
+const { cleanup, temporaryDirectory } = createApiTestFixtures();
afterEach(cleanup);
async function runtimeDirectories() {
@@ -1399,7 +1402,7 @@ describe("CodexSecurity orchestration", () => {
"runs %s scans without signing in to OpenAI",
async (_name, provider, apiKey, model, providerConfig) => {
const { root, repository, codexHome, scanDir } = await scanDirectories();
- const createCodex = mock(completedCodex(root, copyCompletedScan));
+ const createCodex = mock(completedCodex(root));
const onAuthentication = mock<(selected: ScanAuthentication) => void>();
const competingApiKey =
provider === "openrouter" ? "FIREWORKS_API_KEY" : "OPENROUTER_API_KEY";
@@ -1467,7 +1470,7 @@ describe("CodexSecurity orchestration", () => {
"runs Amazon Bedrock scans through %s without signing in to OpenAI",
async (_name, credentials, source) => {
const { root, repository, codexHome, scanDir } = await scanDirectories();
- const createCodex = mock(completedCodex(root, copyCompletedScan));
+ const createCodex = mock(completedCodex(root));
const onAuthentication = mock<(selected: ScanAuthentication) => void>();
const environment = {
OPENAI_API_KEY: "synthetic-openai-key",
@@ -1544,7 +1547,7 @@ describe("CodexSecurity orchestration", () => {
test("uses the selected Bedrock profile for authentication and cost limits", async () => {
const { root, repository, codexHome, scanDir } = await scanDirectories();
- const createCodex = mock(completedCodex(root, copyCompletedScan));
+ const createCodex = mock(completedCodex(root));
const onAuthentication = mock<(selected: ScanAuthentication) => void>();
let savedRecipe: Record<string, unknown> | undefined;
const environment = {
@@ -3110,7 +3113,7 @@ describe("CodexSecurity orchestration", () => {
}
return mockWorkbench(args, input);
},
- createCodex: completedCodex(root, copyCompletedScan),
+ createCodex: completedCodex(root),
},
);
@@ -3154,7 +3157,7 @@ describe("CodexSecurity orchestration", () => {
}
return mockWorkbench(args, input);
},
- createCodex: completedCodex(root, copyCompletedScan),
+ createCodex: completedCodex(root),
},
);
@@ -3188,7 +3191,7 @@ describe("CodexSecurity orchestration", () => {
commands.push(args[0]!);
return mockWorkbench(args, input);
},
- createCodex: completedCodex(root, copyCompletedScan),
+ createCodex: completedCodex(root),
},
);
@@ -3766,7 +3769,7 @@ describe("CodexSecurity orchestration", () => {
uncertain: [],
};
},
- createCodex: completedCodex(root, copyCompletedScan),
+ createCodex: completedCodex(root),
},
);
@@ -3898,7 +3901,7 @@ describe("CodexSecurity orchestration", () => {
{
...scanRuntimeDependencies(codexHome, scanDir),
runWorkbench: recordingWorkbench(commands),
- createCodex: completedCodex(root, copyCompletedScan),
+ createCodex: completedCodex(root),
},
);
@@ -5983,7 +5986,7 @@ describe("CodexSecurity orchestration", () => {
await mkdir(repository);
await mkdir(codexHome);
await mkdir(scanDir, { mode: 0o700 });
- const createCodex = mock(completedCodex(root, copyCompletedScan));
+ const createCodex = mock(completedCodex(root));
const client = new TestClient(
{},
{
@@ -6079,7 +6082,7 @@ describe("CodexSecurity orchestration", () => {
...pathEnvironment,
};
const originalEnvironment = { ...runtimeEnvironment };
- const createCodex = mock(completedCodex(root, copyCompletedScan));
+ const createCodex = mock(completedCodex(root));
const client = new TestClient(
{},
{
@@ -6178,7 +6181,7 @@ process.exit(2);
`,
);
const fakeCommand = nodeCodex(fakeCodex);
- const createCodex = mock(completedCodex(root, copyCompletedScan));
+ const createCodex = mock(completedCodex(root));
const onAuthentication =
mock<(authentication: ScanAuthentication) => void>();
let pythonEnvironment: Record<string, string | undefined> | undefined;
@@ -6516,7 +6519,7 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s
resolvePluginPython: async () => "/managed/python",
prepareOutputDir: async () => scanDir,
repositoryRevision: async () => "deadbeef",
- createCodex: completedCodex(root, copyCompletedScan),
+ createCodex: completedCodex(root),
},
);
const controller = new AbortController();
@@ -6810,7 +6813,7 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s
await mkdir(repository);
await mkdir(codexHome, { mode: 0o700 });
await mkdir(scanDir, { mode: 0o700 });
- const createCodex = mock(completedCodex(root, copyCompletedScan));
+ const createCodex = mock(completedCodex(root));
const client = new TestClient(
{},
{
diff --git a/sdk/typescript/tests-ts/attack-path-tool-name.test.ts b/sdk/typescript/tests-ts/attack-path-tool-name.test.ts
index 48dbafa1d..796c25eb6 100644
--- a/sdk/typescript/tests-ts/attack-path-tool-name.test.ts
+++ b/sdk/typescript/tests-ts/attack-path-tool-name.test.ts
@@ -1,3 +1,4 @@
+import { parseJsonLines } from "./support/json.js";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -27,11 +28,8 @@ test("keeps model-visible attack-path tool names within the Codex limit", async
);
expect(status, stderr).toBe(0);
- const tools = stdout
- .trim()
- .split("\n")
- .map((line) => JSON.parse(line))
- .find((response) => response.id === 2).result.tools as {
+ const tools = parseJsonLines(stdout).find((response) => response.id === 2)
+ .result.tools as {
name: string;
_meta?: { ui?: { visibility?: string[] } };
}[];
diff --git a/sdk/typescript/tests-ts/candidate-normalizer.test.ts b/sdk/typescript/tests-ts/candidate-normalizer.test.ts
index a5d0fe2d2..7c4d3a05b 100644
--- a/sdk/typescript/tests-ts/candidate-normalizer.test.ts
+++ b/sdk/typescript/tests-ts/candidate-normalizer.test.ts
@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";
import { basename, dirname, join, toNamespacedPath } from "node:path";
import { afterEach, describe, expect, test } from "bun:test";
import { PLUGIN_ROOT } from "./plugin-root.js";
+import { parseJsonLines } from "./support/json.js";
const node = Bun.which("node")!;
const helper = join(PLUGIN_ROOT, "mcp", "helpers.mjs");
@@ -102,10 +103,7 @@ function invoke(
);
}
function ledger(f: Fixture): Row[] {
- return readFileSync(f.output, "utf8")
- .split(/\r?\n/u)
- .filter(Boolean)
- .map((line) => JSON.parse(line) as Row);
+ return parseJsonLines<Row>(readFileSync(f.output, "utf8"));
}
afterEach(() => {
for (const root of roots.splice(0))
diff --git a/sdk/typescript/tests-ts/cli-authentication.test.ts b/sdk/typescript/tests-ts/cli-authentication.test.ts
index f6c223b1c..6d116fa56 100644
--- a/sdk/typescript/tests-ts/cli-authentication.test.ts
+++ b/sdk/typescript/tests-ts/cli-authentication.test.ts
@@ -1,4 +1,4 @@
-import { parseJsonLines } from "./support/json.js";
+import { readJsonLines } from "./support/json.js";
import { resolving } from "./support/promises.js";
import { spawnSync } from "node:child_process";
import { existsSync } from "node:fs";
@@ -1329,12 +1329,7 @@ describe("skill authentication", () => {
},
}),
);
- const records = existsSync(log)
- ? (await readFile(log, "utf8"))
- .trim()
- .split("\n")
- .map((line) => JSON.parse(line))
- : [];
+ const records = existsSync(log) ? await readJsonLines(log) : [];
if (ambientConfig !== undefined) {
expect(await readFile(join(ambientHome, "config.toml"), "utf8")).toBe(
ambientConfig,
@@ -2153,7 +2148,7 @@ describe("skill authentication", () => {
: "Authentication failed using OPENAI_API_KEY",
);
else expect(stderr.text()).toBe("Patch applied. Files changed: 1.\n");
- const requests = parseJsonLines(await readFile(requestLog, "utf8"));
+ const requests = await readJsonLines(requestLog);
const methods = requests.map((request) => request.method);
if (!loginFailure) {
expect(
diff --git a/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts b/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts
index 8d2bc7da1..e9ca03545 100644
--- a/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts
+++ b/sdk/typescript/tests-ts/deep-scan-reducer-recovery.test.ts
@@ -1,3 +1,4 @@
+import { parseJsonLines } from "./support/json.js";
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -85,11 +86,9 @@ test("advertises distinct Standard worker and Deep reducer contracts", async ()
new Response(child.stderr).text(),
]);
expect(status, stderr).toBe(0);
- const response = stdout
- .trim()
- .split("\n")
- .map((line) => JSON.parse(line) as { id?: number; result?: unknown })
- .find((message) => message.id === 2)?.result as
+ const response = parseJsonLines<{ id?: number; result?: unknown }>(
+ stdout,
+ ).find((message) => message.id === 2)?.result as
| {
tools: Array<{
name: string;
diff --git a/sdk/typescript/tests-ts/finding-workflow-integration.test.ts b/sdk/typescript/tests-ts/finding-workflow-integration.test.ts
index 4f99a6f2d..d8c2f57b4 100644
--- a/sdk/typescript/tests-ts/finding-workflow-integration.test.ts
+++ b/sdk/typescript/tests-ts/finding-workflow-integration.test.ts
@@ -25,19 +25,16 @@ import {
import { PLUGIN_ROOT } from "./plugin-root.js";
import { DeduplicationReviewError } from "../src/errors.js";
import { workflowFixture } from "./support/workflow-fixture.js";
+import { createTemporaryDirectories } from "./support/temporary-directories.js";
import { rejecting } from "./support/errors.js";
import { capture } from "./cli-fixtures.js";
-const fixtures: Array<Awaited<ReturnType<typeof workflowFixture>>> = [];
-afterEach(async () => {
- await Promise.all(
- fixtures.splice(0).map((fixture) => fixture[Symbol.asyncDispose]()),
- );
-});
+const temporaryDirectories = createTemporaryDirectories();
+afterEach(temporaryDirectories.cleanup);
async function fixture() {
const value = await workflowFixture();
- fixtures.push(value);
+ temporaryDirectories.track(value.root);
const { environment, document, scanDir, repository } = value;
const workbenchOptions = {
environment,
diff --git a/sdk/typescript/tests-ts/plugin-root.ts b/sdk/typescript/tests-ts/plugin-root.ts
index d38d42e8a..cce6d818d 100644
--- a/sdk/typescript/tests-ts/plugin-root.ts
+++ b/sdk/typescript/tests-ts/plugin-root.ts
@@ -1,5 +1,6 @@
import { existsSync } from "node:fs";
-import { cp, readFile } from "node:fs/promises";
+import { chmod, cp, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { brotliDecompressSync } from "node:zlib";
@@ -30,3 +31,11 @@ export const copyCompletedScanFixture = (destination: string) =>
cp(new URL("examples/completed-scan/", bundledPlugin), destination, {
recursive: true,
});
+
+export async function copyCompletedScan(root: string): Promise<string> {
+ const scanDir = join(root, "scan");
+ await copyCompletedScanFixture(scanDir);
+ await chmod(scanDir, 0o700);
+ await writeFile(join(scanDir, "report.md"), "# Scan report\n");
+ return scanDir;
+}
diff --git a/sdk/typescript/tests-ts/sdk-scan-prompts.test.ts b/sdk/typescript/tests-ts/sdk-scan-prompts.test.ts
index 867417ba8..e688de700 100644
--- a/sdk/typescript/tests-ts/sdk-scan-prompts.test.ts
+++ b/sdk/typescript/tests-ts/sdk-scan-prompts.test.ts
@@ -10,8 +10,7 @@ import { completedCodex, preparedRuntime } from "./support/api-events.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { rejecting } from "./support/errors.js";
-const { cleanup, copyCompletedScan, temporaryDirectory } =
- createApiTestFixtures();
+const { cleanup, temporaryDirectory } = createApiTestFixtures();
afterEach(cleanup);
test.each([
@@ -46,7 +45,7 @@ test.each([
recipe = JSON.parse(input!).recipe;
return mockWorkbench(args, input);
},
- createCodex: completedCodex(root, copyCompletedScan, "thread-1"),
+ createCodex: completedCodex(root, "thread-1"),
},
);
await client.run(
diff --git a/sdk/typescript/tests-ts/support/api-events.ts b/sdk/typescript/tests-ts/support/api-events.ts
index 5fa3df61d..f22b01223 100644
--- a/sdk/typescript/tests-ts/support/api-events.ts
+++ b/sdk/typescript/tests-ts/support/api-events.ts
@@ -1,7 +1,7 @@
import type { CodexOptions, ThreadEvent } from "@openai/codex-sdk";
import { CodexSecurity, runScanEvents } from "../../src/api.js";
import type { ScanOptions } from "../../src/index.js";
-import { PLUGIN_ROOT } from "../plugin-root.js";
+import { PLUGIN_ROOT, copyCompletedScan } from "../plugin-root.js";
type PreparedRuntime = Awaited<
ReturnType<
@@ -100,11 +100,7 @@ export async function* failedEvents(): AsyncGenerator<ThreadEvent> {
};
}
-export function completedCodex(
- root: string,
- copyCompletedScan: (root: string) => Promise<string>,
- threadId: string | null = null,
-) {
+export function completedCodex(root: string, threadId: string | null = null) {
return (_options: CodexOptions) => ({
startThread: () => ({
id: threadId,
@@ -128,6 +124,4 @@ export function codexFactory<Run>(runStreamed: Run) {
});
}
-export { createApiTestFixtures } from "./temporary-directories.js";
-
export const failedPostScanEvents = failedEvents;
diff --git a/sdk/typescript/tests-ts/support/security-policy.ts b/sdk/typescript/tests-ts/support/security-policy.ts
index 6ceb791c9..d7e62c89a 100644
--- a/sdk/typescript/tests-ts/support/security-policy.ts
+++ b/sdk/typescript/tests-ts/support/security-policy.ts
@@ -1,18 +1,20 @@
import { execFileSync } from "node:child_process";
-import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { join } from "node:path";
import {
inspectSecurityPolicySources,
readSecurityPolicySnapshot,
resolveSecurityPolicyTarget,
runSecurityPolicyStages,
- type SecurityPolicyDraft,
type SecurityPolicyOptions,
type SecurityPolicyStage,
type SecurityPolicyStageResult,
} from "../../src/security-policy.js";
import { PLUGIN_ROOT } from "../plugin-root.js";
-import { temporaryDirectory } from "./temporary-directories.js";
+import {
+ createTemporaryDirectories,
+ temporaryDirectory,
+} from "./temporary-directories.js";
export const POLICY =
"# Security Policy\n\n## Security Invariants\n\nRequests must be authorized before reading another account's records.\n";
@@ -75,66 +77,66 @@ export function stageResult(
};
}
-export async function policyFixture(): Promise<{
- root: string;
- repository: string;
- outputDir: string;
- generate(options?: {
- path?: string;
- pluginPath?: string;
- run?: (
- stage: SecurityPolicyStage,
- prompt: string,
- ) => Promise<SecurityPolicyStageResult>;
- answerQuestions?: SecurityPolicyOptions["answerQuestions"];
- onWarning?: (warning: string) => void;
- signal?: AbortSignal;
- }): Promise<SecurityPolicyDraft>;
- cleanup(): Promise<void>;
-}> {
- const root = await temporaryDirectory("codex-security-policy-", true);
- const repository = join(root, "repository");
- const outputDir = join(root, "policy");
- await mkdir(repository);
- await mkdir(outputDir, { mode: 0o700 });
+export function createPolicyTestFixtures() {
+ const directories = createTemporaryDirectories();
return {
- root,
- repository,
- outputDir,
- generate: async (options = {}) => {
- const target = await resolveSecurityPolicyTarget(
+ async fixture() {
+ const root = await temporaryDirectory("codex-security-policy-", true);
+ const repository = join(root, "repository");
+ const outputDir = join(root, "policy");
+ await mkdir(repository);
+ await mkdir(outputDir, { mode: 0o700 });
+ directories.track(root);
+ return {
+ root,
repository,
- options.path,
- );
- const sources = await inspectSecurityPolicySources(
- target,
- options.signal,
- );
- return await runSecurityPolicyStages({
- target,
- snapshot: await readSecurityPolicySnapshot(
- target,
- options.signal,
- sources.gitMetadataPaths,
- ),
- policyPaths: sources.policyPaths,
- gitMetadataPaths: sources.gitMetadataPaths,
outputDir,
- pluginRoot: PLUGIN_ROOT,
- pluginPath: options.pluginPath,
- guidance: "Synthetic inherited guidance",
- revision: null,
- model: "gpt-5.6-sol",
- reasoningEffort: "high",
- pluginVersion: "0.1.0",
- signal: options.signal ?? new AbortController().signal,
- run: options.run ?? (async (stage) => stageResult(stage)),
- answerQuestions: options.answerQuestions,
- onWarning: options.onWarning,
- cost: () => null,
- });
+ generate: async (
+ options: {
+ path?: string;
+ run?: (
+ stage: SecurityPolicyStage,
+ prompt: string,
+ ) => Promise<SecurityPolicyStageResult>;
+ answerQuestions?: SecurityPolicyOptions["answerQuestions"];
+ onWarning?: (warning: string) => void;
+ signal?: AbortSignal;
+ } = {},
+ ) => {
+ const target = await resolveSecurityPolicyTarget(
+ repository,
+ options.path,
+ );
+ const sources = await inspectSecurityPolicySources(
+ target,
+ options.signal,
+ );
+ return await runSecurityPolicyStages({
+ target,
+ snapshot: await readSecurityPolicySnapshot(
+ target,
+ options.signal,
+ sources.gitMetadataPaths,
+ ),
+ policyPaths: sources.policyPaths,
+ gitMetadataPaths: sources.gitMetadataPaths,
+ outputDir,
+ pluginRoot: PLUGIN_ROOT,
+ guidance: "Synthetic inherited guidance",
+ revision: null,
+ model: "gpt-5.6-sol",
+ reasoningEffort: "high",
+ pluginVersion: "0.1.0",
+ signal: options.signal ?? new AbortController().signal,
+ run: options.run ?? (async (stage) => stageResult(stage)),
+ answerQuestions: options.answerQuestions,
+ onWarning: options.onWarning,
+ cost: () => null,
+ });
+ },
+ };
},
- cleanup: async () => rm(root, { recursive: true, force: true }),
+ cleanup: directories.cleanup,
};
}
@@ -152,17 +154,3 @@ export async function policyPlugin(
await writeFile(join(plugin, "mcp", "helpers.mjs"), script);
return plugin;
}
-
-export function createPolicyTestFixtures() {
- const fixtures: Awaited<ReturnType<typeof policyFixture>>[] = [];
- return {
- async fixture() {
- const fixture = await policyFixture();
- fixtures.push(fixture);
- return fixture;
- },
- async cleanup(): Promise<void> {
- await Promise.all(fixtures.splice(0).map((fixture) => fixture.cleanup()));
- },
- };
-}
diff --git a/sdk/typescript/tests-ts/support/temporary-directories.ts b/sdk/typescript/tests-ts/support/temporary-directories.ts
index 27b1cfad0..cf7b0740e 100644
--- a/sdk/typescript/tests-ts/support/temporary-directories.ts
+++ b/sdk/typescript/tests-ts/support/temporary-directories.ts
@@ -1,7 +1,3 @@
-import { chmod, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-
-import { copyCompletedScanFixture } from "../plugin-root.js";
import {
createTemporaryDirectories,
temporaryDirectory as createPluginTemporaryDirectory,
@@ -17,13 +13,6 @@ export function createApiTestFixtures(
return {
temporaryDirectories,
cleanup: temporaryDirectories.cleanup,
- async copyCompletedScan(root: string): Promise<string> {
- const scanDir = join(root, "scan");
- await copyCompletedScanFixture(scanDir);
- await chmod(scanDir, 0o700);
- await writeFile(join(scanDir, "report.md"), "# Scan report\n");
- return scanDir;
- },
temporaryDirectory(directoryPrefix = prefix): Promise<string> {
return temporaryDirectories.create(directoryPrefix);
},
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
index 346d3fba2..c2f4cbf86 100644
--- a/sdk/typescript/tests-ts/test-shards.test.ts
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -40,15 +40,6 @@ test("balances measured work and includes new files without mutating the invento
).toEqual(shards);
});
-test.each([0, -1, 1.5, Number.NaN])(
- "rejects an invalid shard count %p",
- (count) => {
- expect(() => shardTestFiles(["a.test.ts"], count)).toThrow(
- "positive integer",
- );
- },
-);
-
test.each([
["unix", 3],
["windows", 7],