Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(cli): share fixtures for terminal interaction and scan history (#2216)

Michael D'Angelof7f76b3a625f
diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts
index 15abb9a12..45bcab461 100644
--- a/sdk/typescript/tests-ts/cli-workbench.test.ts
+++ b/sdk/typescript/tests-ts/cli-workbench.test.ts
@@ -500,37 +500,12 @@ describe("CLI workbench", () => {
{ scan: { scanId: "legacy" } },
{ scanId: "legacy" },
],
- [
- [
- "scans",
- "compare",
- "before",
- "after",
- "--model",
- "synthetic-model",
- "--effort",
- "high",
- "--json",
- ],
- [
- "compare-scans",
- "--before-scan-id",
- "before",
- "--after-scan-id",
- "after",
- "--include-matching-inputs",
- ],
- {
- matchingCached: true,
- matchingInputs: { before: [], after: [] },
- summary: { persisting: 1, resolved: 1 },
- },
- { summary: { persisting: 1, resolved: 1 } },
- ],
- [
+ ];
+ for (const command of ["compare", "match"]) {
+ cases.push([
[
"scans",
- "match",
+ command,
"before",
"after",
"--model",
@@ -553,8 +528,8 @@ describe("CLI workbench", () => {
summary: { persisting: 1, resolved: 1 },
},
{ summary: { persisting: 1, resolved: 1 } },
- ],
- ];
+ ]);
+ }
for (const [argv, expected, response, output] of cases) {
const calls = mock((_args: readonly string[]) => {
return response;
diff --git a/sdk/typescript/tests-ts/patch-tui.test.ts b/sdk/typescript/tests-ts/patch-tui.test.ts
index 33ebd44fe..edcd364e6 100644
--- a/sdk/typescript/tests-ts/patch-tui.test.ts
+++ b/sdk/typescript/tests-ts/patch-tui.test.ts
@@ -81,6 +81,19 @@ function findings(severities: readonly SeverityLevel[]): Finding[] {
return result.findings.findings;
}
+function renderSelection(inputFindings: Finding[]) {
+ const selected: (PatchSelection | null)[] = [];
+ const app = render(
+ createElement(PatchTui, {
+ repository: "/work/example",
+ findings: inputFindings,
+ color: false,
+ onComplete: (value) => selected.push(value),
+ }),
+ );
+ return { app, selected };
+}
+
async function press(
app: ReturnType<typeof render>,
input: string,
@@ -109,15 +122,7 @@ describe("interactive patch finding browser", () => {
"high",
]);
const originalOrder = inputFindings.map(({ occurrenceId }) => occurrenceId);
- const selected: (PatchSelection | null)[] = [];
- const app = render(
- createElement(PatchTui, {
- repository: "/work/example",
- findings: inputFindings,
- color: false,
- onComplete: (value) => selected.push(value),
- }),
- );
+ const { app, selected } = renderSelection(inputFindings);
const displayedFindings = (app.lastFrame() ?? "")
.split("\n")
@@ -288,14 +293,8 @@ describe("interactive patch finding browser", () => {
});
test("combines severity presets with individual finding selection", async () => {
- const selected: (PatchSelection | null)[] = [];
- const app = render(
- createElement(PatchTui, {
- repository: "/work/example",
- findings: findings(["high", "medium", "low"]),
- color: false,
- onComplete: (value) => selected.push(value),
- }),
+ const { app, selected } = renderSelection(
+ findings(["high", "medium", "low"]),
);
await press(app, "2");
@@ -313,15 +312,7 @@ describe("interactive patch finding browser", () => {
});
test("edits instructions per finding and only returns selected guidance", async () => {
- const selected: (PatchSelection | null)[] = [];
- const app = render(
- createElement(PatchTui, {
- repository: "/work/example",
- findings: findings(["high", "medium"]),
- color: false,
- onComplete: (value) => selected.push(value),
- }),
- );
+ const { app, selected } = renderSelection(findings(["high", "medium"]));
await press(app, "i");
expect(app.lastFrame()).toContain("Enter save");
@@ -357,15 +348,7 @@ describe("interactive patch finding browser", () => {
});
test("optionally creates a draft pull request after selected patches", async () => {
- const selected: (PatchSelection | null)[] = [];
- const app = render(
- createElement(PatchTui, {
- repository: "/work/example",
- findings: findings(["high"]),
- color: false,
- onComplete: (value) => selected.push(value),
- }),
- );
+ const { app, selected } = renderSelection(findings(["high"]));
expect(app.lastFrame()).toContain(
"[ ] Create draft pull request or merge request after patching",
@@ -386,15 +369,7 @@ describe("interactive patch finding browser", () => {
});
test("cancels and clears finding instructions without leaving the browser", async () => {
- const selected: (PatchSelection | null)[] = [];
- const app = render(
- createElement(PatchTui, {
- repository: "/work/example",
- findings: findings(["high"]),
- color: false,
- onComplete: (value) => selected.push(value),
- }),
- );
+ const { app, selected } = renderSelection(findings(["high"]));
await press(app, "i");
await press(app, "Discard this guidance.");
@@ -413,15 +388,7 @@ describe("interactive patch finding browser", () => {
});
test("deletes complete characters from finding instructions", async () => {
- const selected: (PatchSelection | null)[] = [];
- const app = render(
- createElement(PatchTui, {
- repository: "/work/example",
- findings: findings(["high"]),
- color: false,
- onComplete: (value) => selected.push(value),
- }),
- );
+ const { app, selected } = renderSelection(findings(["high"]));
await press(app, "i");
await press(app, "Ask 👩‍💻");
@@ -442,15 +409,7 @@ describe("interactive patch finding browser", () => {
test("allows selecting none and canceling without patching", async () => {
for (const input of ["q", "\r"]) {
- const selected: (PatchSelection | null)[] = [];
- const app = render(
- createElement(PatchTui, {
- repository: "/work/example",
- findings: findings(["high"]),
- color: false,
- onComplete: (value) => selected.push(value),
- }),
- );
+ const { app, selected } = renderSelection(findings(["high"]));
if (input === "\r") {
await press(app, "n");
expect(app.lastFrame()).toContain("0/1 selected");
diff --git a/sdk/typescript/tests-ts/scan-dashboard.test.ts b/sdk/typescript/tests-ts/scan-dashboard.test.ts
index 4966fdd77..43416bf7f 100644
--- a/sdk/typescript/tests-ts/scan-dashboard.test.ts
+++ b/sdk/typescript/tests-ts/scan-dashboard.test.ts
@@ -32,6 +32,29 @@ function createDashboard(
});
}
+function budgetRequest(signal = new AbortController().signal) {
+ return {
+ maxCostUsd: 20,
+ cost: fakeResult([], "complete", {
+ input_tokens: 100,
+ output_tokens: 1,
+ }).cost!,
+ signal,
+ };
+}
+
+function recordActions(dashboard: ScanDashboard, count: number): void {
+ for (let index = 1; index <= count; index += 1) {
+ dashboard.record({
+ id: `action-${index}`,
+ kind: "command",
+ status: "completed",
+ description: `rg -n finding-${index} routes.ts`,
+ paths: [],
+ });
+ }
+}
+
function lastFrame(stderr: ReturnType<typeof capture>): string {
return stripVTControlCharacters(stderr.text())
.split("CODEX SECURITY · juice-shop")
@@ -120,14 +143,7 @@ describe("live scan dashboard", () => {
dashboard.start();
try {
input.emit("data", "d");
- const answer = dashboard.requestBudgetIncrease({
- maxCostUsd: 20,
- cost: fakeResult([], "complete", {
- input_tokens: 100,
- output_tokens: 1,
- }).cost!,
- signal: new AbortController().signal,
- });
+ const answer = dashboard.requestBudgetIncrease(budgetRequest());
input.emit("data", "\u001BO");
input.emit("data", "1;5P\u001B\u001B[A");
await expect(answer).resolves.toBeUndefined();
@@ -150,14 +166,7 @@ describe("live scan dashboard", () => {
onInterrupt: interrupted,
});
dashboard.start();
- const answer = dashboard.requestBudgetIncrease({
- maxCostUsd: 20,
- cost: fakeResult([], "complete", {
- input_tokens: 100,
- output_tokens: 1,
- }).cost!,
- signal: new AbortController().signal,
- });
+ const answer = dashboard.requestBudgetIncrease(budgetRequest());
input.emit("data", prefix);
expect(lastFrame(stderr)).toContain("Raise total USD limit");
input.emit("data", key);
@@ -192,14 +201,7 @@ describe("live scan dashboard", () => {
dashboard.start();
try {
input.emit("data", "d");
- const answer = dashboard.requestBudgetIncrease({
- maxCostUsd: 20,
- cost: fakeResult([], "complete", {
- input_tokens: 100,
- output_tokens: 1,
- }).cost!,
- signal: new AbortController().signal,
- });
+ const answer = dashboard.requestBudgetIncrease(budgetRequest());
for (const chunk of chunks) input.emit("data", chunk);
expect(lastFrame(stderr)).not.toContain("Raise total USD limit");
await expect(answer).resolves.toBeUndefined();
@@ -980,14 +982,7 @@ describe("live scan dashboard", () => {
const dashboard = createDashboard(capture(true).stream, { input });
dashboard.start();
try {
- const answer = dashboard.requestBudgetIncrease({
- maxCostUsd: 20,
- cost: fakeResult([], "complete", {
- input_tokens: 100,
- output_tokens: 1,
- }).cost!,
- signal: new AbortController().signal,
- });
+ const answer = dashboard.requestBudgetIncrease(budgetRequest());
input.emit("data", "30");
const prefix = "\u001B".repeat(escapes);
const chunks = coalesced
@@ -1102,14 +1097,7 @@ describe("live scan dashboard", () => {
dashboard.start();
try {
input.emit("data", sequence);
- const answer = dashboard.requestBudgetIncrease({
- maxCostUsd: 20,
- cost: fakeResult([], "complete", {
- input_tokens: 100,
- output_tokens: 1,
- }).cost!,
- signal: new AbortController().signal,
- });
+ const answer = dashboard.requestBudgetIncrease(budgetRequest());
await Promise.resolve();
expect(jest.getTimerCount()).toBe(0);
jest.runAllTimers();
@@ -1140,14 +1128,9 @@ describe("live scan dashboard", () => {
try {
for (let index = 0; index < 20; index++)
dashboard.note(`Activity ${index}`);
- const answer = dashboard.requestBudgetIncrease({
- maxCostUsd: 20,
- cost: fakeResult([], "complete", {
- input_tokens: 100,
- output_tokens: 1,
- }).cost!,
- signal: controller.signal,
- });
+ const answer = dashboard.requestBudgetIncrease(
+ budgetRequest(controller.signal),
+ );
input.emit("data", prefix);
controller.abort();
await expect(answer).resolves.toBeUndefined();
@@ -1264,14 +1247,7 @@ describe("live scan dashboard", () => {
}
if (mode === "budget")
void dashboard
- .requestBudgetIncrease({
- maxCostUsd: 20,
- cost: fakeResult([], "complete", {
- input_tokens: 100,
- output_tokens: 1,
- }).cost!,
- signal: controller.signal,
- })
+ .requestBudgetIncrease(budgetRequest(controller.signal))
.then((value) => {
answer = value;
});
@@ -2003,15 +1979,7 @@ describe("live scan dashboard", () => {
dashboard.start();
expect(input.isRaw).toBe(true);
- for (let index = 1; index <= 20; index += 1) {
- dashboard.record({
- id: `action-${index}`,
- kind: "command",
- status: "completed",
- description: `rg -n finding-${index} routes.ts`,
- paths: [],
- });
- }
+ recordActions(dashboard, 20);
let frame = lastFrame(stderr);
expect(frame).toContain("finding-16");
@@ -2066,15 +2034,7 @@ describe("live scan dashboard", () => {
);
dashboard.start();
- for (let index = 1; index <= 30; index += 1) {
- dashboard.record({
- id: `action-${index}`,
- kind: "command",
- status: "completed",
- description: `rg -n finding-${index} routes.ts`,
- paths: [],
- });
- }
+ recordActions(dashboard, 30);
const framesBefore = stderr.text().match(/\u001B\[H/gu)?.length ?? 0;
input.emit("data", Buffer.from("\u001B[A".repeat(6)));
@@ -2103,15 +2063,7 @@ describe("live scan dashboard", () => {
);
dashboard.start();
- for (let index = 1; index <= 20; index += 1) {
- dashboard.record({
- id: `action-${index}`,
- kind: "command",
- status: "completed",
- description: `rg -n finding-${index} routes.ts`,
- paths: [],
- });
- }
+ recordActions(dashboard, 20);
input.emit("data", "\u0015");
let frame = lastFrame(stderr);