Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: batch workbench setup for saved-scan fixtures (#1901)

Michael D'Angelo2d457e32917b
diff --git a/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts b/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts
index a3f586fbe..79d4527bf 100644
--- a/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts
+++ b/sdk/typescript/tests-ts/saved-scan-bootstrap.test.ts
@@ -20,18 +20,20 @@ import { resolvePluginPython, runWorkbench } from "../src/runtime.js";
import { SqliteFindingsStore } from "../src/server/sqlite-store.js";
import type { FindingEmbeddingBinding } from "../src/deduplication/local.js";
import type { Finding } from "../src/models.js";
+import type { JsonObject } from "../src/config.js";
import type { CodexReview } from "../src/deduplication/codex-review.js";
import { screeningPairSlot } from "../src/deduplication/deduplication-reviewer.js";
import { copyCompletedScanFixture, PLUGIN_ROOT } from "./plugin-root.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
import { rejecting } from "./support/errors.js";
+import { runWorkbenchFixtureCommands } from "./support/workbench-fixtures.js";
const { temporaryDirectory, cleanup } = createApiTestFixtures(
- "saved-scan-bootstrap-",
+ "saved-scan-bootstrap-résumé-",
);
afterEach(cleanup);
-async function fixture(git = false) {
+async function fixture(git = false, separateSetupCommands = false) {
const root = await temporaryDirectory();
const repository = join(root, "repository");
await mkdir(join(repository, "src"), { recursive: true });
@@ -70,23 +72,26 @@ async function fixture(git = false) {
};
const python = await resolvePluginPython({ environment });
const options = { environment, python, pluginRoot: PLUGIN_ROOT };
- async function scan(name: string) {
- const scanDir = join(root, name);
- await mkdir(scanDir, { mode: 0o700 });
- const registered = await runWorkbench(options, [
- "register-cli-scan",
- "--repository",
+ const registrationArguments = (scanDir: string) => [
+ "register-cli-scan",
+ "--repository",
+ repository,
+ "--scan-dir",
+ scanDir,
+ "--recipe-json",
+ JSON.stringify({
+ config: {},
+ mode: "standard",
repository,
- "--scan-dir",
- scanDir,
- "--recipe-json",
- JSON.stringify({
- config: {},
- mode: "standard",
- repository,
- target: { kind: "repository", paths: [] },
- }),
- ]);
+ target: { kind: "repository", paths: [] },
+ }),
+ ];
+ async function scan(name: string, registered?: JsonObject) {
+ const scanDir = join(root, name);
+ if (registered === undefined) {
+ await mkdir(scanDir, { mode: 0o700 });
+ registered = await runWorkbench(options, registrationArguments(scanDir));
+ }
const scanId = registered["scanId"] as string;
await copyCompletedScanFixture(scanDir);
const manifest = JSON.parse(
@@ -136,11 +141,36 @@ async function fixture(git = false) {
);
coverage.scanId = scanId;
await writeFile(join(scanDir, "coverage.json"), JSON.stringify(coverage));
- await runWorkbench(options, ["complete-scan", "--scan-id", scanId]);
+ if (separateSetupCommands)
+ await runWorkbench(options, ["complete-scan", "--scan-id", scanId]);
return { scanId, scanDir };
}
- const first = await scan("first");
- const second = await scan("second");
+ let first: Awaited<ReturnType<typeof scan>>;
+ let second: Awaited<ReturnType<typeof scan>>;
+ if (separateSetupCommands) {
+ first = await scan("first");
+ second = await scan("second");
+ } else {
+ const directories = [join(root, "first"), join(root, "second")];
+ for (const directory of directories)
+ await mkdir(directory, { mode: 0o700 });
+ const registrations = await runWorkbenchFixtureCommands(
+ python,
+ environment,
+ directories.map(registrationArguments),
+ );
+ first = await scan("first", registrations[0]!);
+ second = await scan("second", registrations[1]!);
+ await runWorkbenchFixtureCommands(
+ python,
+ environment,
+ [first, second].map(({ scanId }) => [
+ "complete-scan",
+ "--scan-id",
+ scanId,
+ ]),
+ );
+ }
const embed = mock(async (findings: readonly Finding[]) =>
findings.map(() => ({ model: "synthetic", vector: [1, 0, 0] })),
);
@@ -182,7 +212,8 @@ async function fixture(git = false) {
}
test("historical reviews carry the saved occurrence revision instead of the current checkout", async () => {
- const f = await fixture(true);
+ // Keep an end-to-end registration/completion fixture on separate processes.
+ const f = await fixture(true, true);
const manifest = JSON.parse(
await readFile(join(f.first.scanDir, "scan-manifest.json"), "utf8"),
);
diff --git a/sdk/typescript/tests-ts/scan-resume.test.ts b/sdk/typescript/tests-ts/scan-resume.test.ts
index f5ea9558f..020357bfb 100644
--- a/sdk/typescript/tests-ts/scan-resume.test.ts
+++ b/sdk/typescript/tests-ts/scan-resume.test.ts
@@ -1,5 +1,6 @@
import { createCliTest } from "./support/cli-run.js";
import { workbenchCommand } from "./support/workbench-command.js";
+import { runWorkbenchFixtureCommands } from "./support/workbench-fixtures.js";
import { gitText } from "./support/shell.js";
import { readJsonLines } from "./support/json.js";
import { randomUUID } from "node:crypto";
@@ -49,6 +50,7 @@ async function interruptedScan(
resolvedDeep = false,
modelProvider?: string,
knowledgeState: "saved" | "legacy" = "saved",
+ separateSetupCommands = false,
) {
const root = await temporaryDirectory();
const repository = bulk
@@ -202,13 +204,9 @@ async function interruptedScan(
if (knowledgeSnapshot !== undefined)
await saveScanKnowledge(scanDir, knowledgeSnapshot);
const threadId = randomUUID();
- await command([
- "set-scan-thread",
- "--scan-id",
- scanId,
- "--thread-id",
- threadId,
- ]);
+ const setupCommands = [
+ ["set-scan-thread", "--scan-id", scanId, "--thread-id", threadId],
+ ];
const sessionPath = join(codexHome, "sessions", `rollout-${threadId}.jsonl`);
await writeFile(
sessionPath,
@@ -218,7 +216,7 @@ async function interruptedScan(
}) + "\n",
);
if (mode === "deep") {
- await command([
+ setupCommands.push([
"begin-deep-scan",
"--scan-id",
scanId,
@@ -233,7 +231,7 @@ async function interruptedScan(
const prompt = join(scanDir, "setup-prompt.md");
await writeFile(prompt, "Saved setup prompt.\n");
for (const status of ["running", "succeeded"]) {
- await command([
+ setupCommands.push([
"upsert-deep-scan-worker",
"--scan-id",
scanId,
@@ -250,6 +248,11 @@ async function interruptedScan(
]);
}
}
+ if (separateSetupCommands) {
+ for (const args of setupCommands) await command(args);
+ } else {
+ await runWorkbenchFixtureCommands(python, environment, setupCommands);
+ }
const checkpoint = join(scanDir, "checkpoint.json");
await writeFile(checkpoint, '{"completed":"setup"}\n');
return {
@@ -271,7 +274,16 @@ async function interruptedScan(
}
test("resume resolves an interrupted scan without changing its ID, recipe, or completed workers", async () => {
- const f = await interruptedScan();
+ // Retain the separate-process setup contract alongside the batched fixtures.
+ const f = await interruptedScan(
+ "deep",
+ false,
+ {},
+ false,
+ undefined,
+ "saved",
+ true,
+ );
const before = await f.command([
"get-deep-scan",
"--scan-id",
diff --git a/sdk/typescript/tests-ts/support/workbench-fixtures.ts b/sdk/typescript/tests-ts/support/workbench-fixtures.ts
new file mode 100644
index 000000000..5ed4d7e5e
--- /dev/null
+++ b/sdk/typescript/tests-ts/support/workbench-fixtures.ts
@@ -0,0 +1,46 @@
+import { join } from "node:path";
+import type { JsonObject } from "../../src/config.js";
+import { PLUGIN_ROOT } from "../plugin-root.js";
+import { runCommand } from "./shell.js";
+
+const setupProgram = String.raw`
+import contextlib, io, json, runpy, sys
+script = sys.argv[1]
+commands = json.load(sys.stdin)
+workbench = runpy.run_path(script)
+results = []
+for arguments in commands:
+ sys.argv = [script, *arguments]
+ sys.stdin = io.TextIOWrapper(io.BytesIO(), encoding="utf-8")
+ output = io.StringIO()
+ with contextlib.redirect_stdout(output):
+ workbench["main"]()
+ results.append(json.loads(output.getvalue()))
+print(json.dumps(results))
+`;
+
+/** Run fixture setup through the real parser and transitions, sharing Python startup. */
+export async function runWorkbenchFixtureCommands(
+ python: string,
+ environment: NodeJS.ProcessEnv,
+ commands: readonly (readonly string[])[],
+): Promise<JsonObject[]> {
+ const result = await runCommand(
+ python,
+ [
+ "-I",
+ "-X",
+ "utf8",
+ "-B",
+ "-c",
+ setupProgram,
+ join(PLUGIN_ROOT, "scripts", "workbench_db.py"),
+ ],
+ { env: environment, input: JSON.stringify(commands), timeout: 0 },
+ );
+ if (result.status !== 0)
+ throw new Error(result.stderr || "Workbench fixture setup failed.", {
+ cause: result.error,
+ });
+ return JSON.parse(result.stdout) as JsonObject[];
+}