Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(tooling): remove duplicate benchmark checks and line collection (#1359)

mldangelo-oai2379b181266a
diff --git a/.github/scripts/check_plugin_source_compatibility.mts b/.github/scripts/check_plugin_source_compatibility.mts
index 5fcd6b53f..814292681 100644
--- a/.github/scripts/check_plugin_source_compatibility.mts
+++ b/.github/scripts/check_plugin_source_compatibility.mts
@@ -60,11 +60,10 @@ function lineEndsNaturally(line: string): boolean {
);
}
-function hardWrappedLines(content: string): number[] {
+function* hardWrappedLines(content: string): Generator<number> {
const lines = content.split(
/\r\n|[\n\r\v\f\u001c-\u001e\u0085\u2028\u2029]/u,
);
- const offenders: number[] = [];
let inFence = false;
let inFrontmatter = content.startsWith("---\n");
for (let index = 0; index < lines.length - 1; index++) {
@@ -85,9 +84,8 @@ function hardWrappedLines(content: string): number[] {
if (LIST_ITEM.test(followingLine) || lineEndsNaturally(line)) continue;
if (!/[A-Za-z0-9`]$/u.test(stripped)) continue;
if (!/^[A-Za-z0-9`(]/u.test(followingLine.trim())) continue;
- offenders.push(index + 1);
+ yield index + 1;
}
- return offenders;
}
function sourceCompatibilityErrors(pluginRoot: string): string[] {
diff --git a/evals/triage-finding/sastbench/scripts/install-sastbench.mts b/evals/triage-finding/sastbench/scripts/install-sastbench.mts
index 11535502e..0c6c7be94 100644
--- a/evals/triage-finding/sastbench/scripts/install-sastbench.mts
+++ b/evals/triage-finding/sastbench/scripts/install-sastbench.mts
@@ -6,8 +6,6 @@ import path from "node:path";
import {
DEFAULT_INSTALL_ROOT,
- EXPECTED_CASE_COUNT,
- EXPECTED_LABEL_COUNTS,
SASTBENCH_COMMIT,
SASTBENCH_DATASET_RELATIVE_PATH,
SASTBENCH_DATASET_SHA256,
@@ -85,19 +83,6 @@ export function verifyInstallation(
`SastBench dataset SHA-256 mismatch: expected ${SASTBENCH_DATASET_SHA256}, got ${inspection.datasetSha256}`,
);
}
- if (inspection.caseCount !== EXPECTED_CASE_COUNT) {
- throw new Error(
- `SastBench case count mismatch: expected ${EXPECTED_CASE_COUNT}, got ${inspection.caseCount}`,
- );
- }
- for (const [label, expected] of Object.entries(EXPECTED_LABEL_COUNTS)) {
- const actual = inspection.labelCounts?.[label];
- if (actual !== expected) {
- throw new Error(
- `SastBench ${label} count mismatch: expected ${expected}, got ${actual}`,
- );
- }
- }
return inspection;
}
diff --git a/evals/triage-finding/sastbench/scripts/test-install-sastbench.mts b/evals/triage-finding/sastbench/scripts/test-install-sastbench.mts
index f34993632..1d2ae4388 100644
--- a/evals/triage-finding/sastbench/scripts/test-install-sastbench.mts
+++ b/evals/triage-finding/sastbench/scripts/test-install-sastbench.mts
@@ -42,19 +42,5 @@ assert.throws(
() => verifyInstallation(validInspection({ datasetSha256: "0".repeat(64) })),
/dataset SHA-256 mismatch/,
);
-assert.throws(
- () =>
- verifyInstallation(validInspection({ caseCount: EXPECTED_CASE_COUNT - 1 })),
- /case count mismatch/,
-);
-assert.throws(
- () =>
- verifyInstallation(
- validInspection({
- labelCounts: { true_positive: 300, false_positive: 2437 },
- }),
- ),
- /true_positive count mismatch/,
-);
console.log("sastbench installer verification tests passed");
diff --git a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
index c670bed32..8cdeedce8 100644
--- a/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
+++ b/evals/triage-finding/sastbench/scripts/test-sastbench-lib.mts
@@ -69,6 +69,19 @@ validateDataset([positive, negative], {
labelCounts: { true_positive: 1, false_positive: 1 },
});
+assert.throws(
+ () => validateDataset([positive, negative]),
+ /case count mismatch/,
+);
+assert.throws(
+ () =>
+ validateDataset([positive, negative], {
+ caseCount: 2,
+ labelCounts: { true_positive: 2, false_positive: 0 },
+ }),
+ /true_positive count mismatch/,
+);
+
const sparseScannerRecord = benchmarkRecord({
to_analyzer: {
...positive.to_analyzer,