Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(ci): pass JUnit shard files directly to the comparator (#2250)

Michael D'Angeloc40e7bbf3857
diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml
index a6bd1ba25..de21b4711 100644
--- a/.github/workflows/test-quality.yml
+++ b/.github/workflows/test-quality.yml
@@ -201,8 +201,13 @@ jobs:
for mode in baseline expanded; do
shards=3
if [[ "$os" == windows-latest ]]; then shards=7; fi
- if [[ "$mode" == expanded ]]; then shards=$((shards * 2)); fi
reports=(reports/runner-"$os"-"$mode"-*.xml)
+ if [[ "$mode" == expanded ]]; then
+ shards=$((shards * 2))
+ expanded_reports=("${reports[@]}")
+ else
+ baseline_reports=("${reports[@]}")
+ fi
if (( ${#reports[@]} != shards )); then
echo "Expected $shards reports for $os/$mode, found ${#reports[@]}" >&2
comparison_status=1
@@ -215,7 +220,7 @@ jobs:
fi
done
done
- node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "reports/runner-$os-baseline-*.xml" "reports/runner-$os-expanded-*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
+ node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "${baseline_reports[@]}" -- "${expanded_reports[@]}" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
done
exit "$comparison_status"
diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json
index bcebeadd3..a97b00433 100644
--- a/sdk/typescript/package.json
+++ b/sdk/typescript/package.json
@@ -116,7 +116,6 @@
"ink-testing-library": "4.0.0",
"json-schema-to-typescript": "16.0.0",
"json5": "2.2.3",
- "minimatch": "10.2.6",
"prettier": "3.9.10",
"typescript": "7.0.2"
}
diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yaml
index 0b73b5310..954fb1d2a 100644
--- a/sdk/typescript/pnpm-lock.yaml
+++ b/sdk/typescript/pnpm-lock.yaml
@@ -113,9 +113,6 @@ importers:
json5:
specifier: 2.2.3
version: 2.2.3
- minimatch:
- specifier: 10.2.6
- version: 10.2.6
prettier:
specifier: 3.9.10
version: 3.9.10
diff --git a/sdk/typescript/scripts/test-report-comparison.mts b/sdk/typescript/scripts/test-report-comparison.mts
index 3dbf45769..175bbb7ac 100644
--- a/sdk/typescript/scripts/test-report-comparison.mts
+++ b/sdk/typescript/scripts/test-report-comparison.mts
@@ -1,8 +1,7 @@
// Compare Bun JUnit inventories before changing the required CI runner.
-import { lstatSync, readFileSync, readdirSync, statSync } from "node:fs";
-import { basename, dirname, join, sep } from "node:path";
+import { readFileSync } from "node:fs";
+import { basename } from "node:path";
import { parseArgs } from "node:util";
-import { minimatch } from "minimatch";
import { SaxesParser, type SaxesTagNS } from "saxes";
type Output = Pick<Console, "log" | "error">;
@@ -17,59 +16,6 @@ type TestRecord = {
attributes: SaxesTagNS["attributes"];
status: TestStatus;
};
-function matchingReports(pattern: string): string[] {
- // Python glob treats ** as one component and does not expand braces/extglobs.
- const directoriesOnly =
- pattern.endsWith(sep) ||
- (process.platform === "win32" && pattern.endsWith("/"));
- if (!/[*?[]/u.test(pattern)) {
- try {
- const stat = directoriesOnly ? statSync(pattern) : lstatSync(pattern);
- return !directoriesOnly || stat.isDirectory() ? [pattern] : [];
- } catch {
- return [];
- }
- }
- const parent = dirname(pattern);
- const namePattern = basename(pattern);
- return (/[*?[]/u.test(parent) ? matchingReports(parent) : [parent]).flatMap(
- (directory) => {
- let names: string[];
- try {
- names = readdirSync(directory);
- } catch {
- return [];
- }
- return names
- .filter(
- (name) =>
- (!name.startsWith(".") || namePattern.startsWith(".")) &&
- minimatch(
- name,
- namePattern.replaceAll("\\", "\\\\").replaceAll("[^", "[\\^"),
- {
- dot: true,
- nobrace: true,
- noext: true,
- noglobstar: true,
- nonegate: true,
- nocomment: true,
- nocase: process.platform === "win32",
- },
- ),
- )
- .map((name) => join(directory, name))
- .filter((path) => {
- if (!directoriesOnly) return true;
- try {
- return statSync(path).isDirectory();
- } catch {
- return false;
- }
- });
- },
- );
-}
function integer(value: string): bigint {
if (!/^[+-]?\d(?:_?\d)*$/u.test(value.trim())) {
@@ -163,31 +109,30 @@ function readReport(path: string, output: Output) {
};
}
-function readReports(patterns: string[], output: Output) {
+function readReports(paths: string[], group: string, output: Output) {
+ if (!paths.length) throw new Error(`No ${group} reports supplied`);
const cases = new Map<string, number>();
const durations: number[] = [];
let failed = false;
- for (const pattern of patterns) {
- const paths = matchingReports(pattern).sort((a, b) =>
- Buffer.compare(Buffer.from(a), Buffer.from(b)),
- );
- if (!paths.length) throw new Error(`No reports match ${pattern}`);
- for (const path of paths) {
- const report = readReport(path, output);
- failed ||= report.failed;
- for (const [identity, count] of report.cases)
- cases.set(identity, (cases.get(identity) ?? 0) + count);
- durations.push(report.duration);
- }
+ for (const path of paths) {
+ const report = readReport(path, output);
+ failed ||= report.failed;
+ for (const [identity, count] of report.cases)
+ cases.set(identity, (cases.get(identity) ?? 0) + count);
+ durations.push(report.duration);
}
return { cases, durations, failed };
}
-function compareGroups(patterns: string[], output: Output): number {
+function compareGroups(
+ baselinePaths: string[],
+ candidatePaths: string[],
+ output: Output,
+): number {
output.log("| Report | Cases | Skipped | Seconds |");
output.log("| --- | ---: | ---: | ---: |");
- const baseline = readReports([patterns[0]!], output);
- const candidate = readReports(patterns.slice(1), output);
+ const baseline = readReports(baselinePaths, "baseline", output);
+ const candidate = readReports(candidatePaths, "candidate", output);
let failed = baseline.failed || candidate.failed;
for (const [label, left, right] of [
["Missing", baseline.cases, candidate.cases],
@@ -233,29 +178,34 @@ export function compareTestReports(
argv: string[],
output: Output = console,
): number {
+ const separator = argv.indexOf("--");
let args: { values: { help?: boolean }; positionals: string[] };
try {
args = parseArgs({
- args: argv,
+ args: separator === -1 ? argv : argv.slice(0, separator),
options: { help: { type: "boolean", short: "h" } },
allowPositionals: true,
});
- if (!args.values.help && args.positionals.length < 2)
- throw new Error(
- "a baseline and at least one candidate report are required",
- );
} catch (error) {
output.error((error as Error).message);
return 2;
}
if (args.values.help) {
output.log(
- "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nBaseline and candidates are JUnit files or glob patterns.",
+ "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidate [candidate ...]\n or: node --experimental-strip-types compare-test-reports.mts baseline [baseline ...] -- candidate [candidate ...]\nPass JUnit filenames. Your shell can expand wildcard patterns.",
);
return 0;
}
+ if (separator === -1 && args.positionals.length < 2) {
+ output.error("a baseline and at least one candidate report are required");
+ return 2;
+ }
+ const baselinePaths =
+ separator === -1 ? args.positionals.slice(0, 1) : args.positionals;
+ const candidatePaths =
+ separator === -1 ? args.positionals.slice(1) : argv.slice(separator + 1);
try {
- return compareGroups(args.positionals, output);
+ return compareGroups(baselinePaths, candidatePaths, output);
} catch (error) {
output.error((error as Error).message);
return 1;
diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts
index e7a5c7ce7..58955b8e4 100644
--- a/sdk/typescript/tests-ts/test-reports.test.ts
+++ b/sdk/typescript/tests-ts/test-reports.test.ts
@@ -137,12 +137,11 @@ describe("JUnit inventory comparison", () => {
test("runs every workflow comparison before reporting a mismatch", async () => {
const fixture = await workflowFixtures();
const expected = ["ubuntu-latest", "macos-latest", "windows-latest"].map(
- (os) =>
- `reports/runner-${os}-baseline-*.xml|reports/runner-${os}-expanded-*.xml`,
+ (os) => `reports/runner-${os}-baseline-1.xml`,
);
const mock = `node() {
- printf '%s\\n' "$3|$4"
- [[ "$3|$4" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]
+ printf '%s\\n' "$3"
+ [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]
}`;
for (const failedReport of ["", expected[0]!]) {
const { status, stderr, summary } = await runWorkflow(fixture, mock, {
@@ -166,7 +165,7 @@ describe("JUnit inventory comparison", () => {
}
const { status, stderr, summary } = await runWorkflow(
fixture,
- 'node() { command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$3" "$4"; }',
+ 'node() { shift 2; command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$@"; }',
{ CODEX_SECURITY_TEST_COMPARATOR: comparator },
);
expect(status, stderr).toBe(incomplete ? 1 : 0);
@@ -227,7 +226,7 @@ describe("JUnit inventory comparison", () => {
]);
const { status, stderr, summary } = await runWorkflow(
fixture,
- `node() { printf '%s\\n' "$3|$4"; }`,
+ "node() { echo compared; }",
);
expect(status, stderr).toBe(1);
expect(stderr).toContain(
@@ -245,12 +244,9 @@ describe("JUnit inventory comparison", () => {
const passed = testcase("accepts &amp; preserves");
const skipped = testcase("platform-only", "<skipped/>");
const baseline = await fixture.report("baseline.xml", [passed, skipped]);
- await fixture.report("shard-1.xml", [skipped]);
- await fixture.report("shard-2.xml", [passed]);
- const result = await compareCli(
- baseline,
- join(fixture.root, "shard-*.xml"),
- );
+ const first = await fixture.report("shard-1.xml", [skipped]);
+ const second = await fixture.report("shard-2.xml", [passed]);
+ const result = await compareCli(baseline, first, second);
expect(result.status, result.stderr).toBe(0);
expect(result.stderr).toMatch(typeStrippingNotice);
expect(result.stdout).toBe(
@@ -301,40 +297,40 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
);
});
- test("preserves file, character-class, hidden-file, and nonrecursive glob matching", async () => {
+ test("reads literal report paths containing spaces and brackets", async () => {
const fixture = await fixtures();
- const cases = [testcase("portable")];
+ const cases = [testcase("selected")];
const baseline = await fixture.report("baseline.xml", cases);
- const directory = join(fixture.root, "reports with spaces");
- await mkdir(directory);
- await fixture.report("reports with spaces/shard-a.xml", cases);
- await fixture.report("reports with spaces/.hidden.xml", [
- testcase("hidden"),
- ]);
- await mkdir(join(directory, "nested"));
- await fixture.report("reports with spaces/nested/shard-b.xml", [
- testcase("nested"),
- ]);
- for (const pattern of [
- "reports with spaces/shard-?.xml",
- "reports with spaces/shard-[ab].xml",
- "reports with spaces/shard-[!b].xml",
- "reports with spaces/*.xml",
- "**/shard-*.xml",
+ const literal = await fixture.report("shard [ab].xml", cases);
+ await fixture.report("shard a.xml", [testcase("unrelated")]);
+ const result = compare(baseline, literal);
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain("| shard [ab].xml |");
+
+ const absent = join(fixture.root, "shard *.xml");
+ for (const args of [
+ [baseline, absent],
+ [absent, baseline],
]) {
- const result = compare(baseline, join(fixture.root, pattern));
- expect(result.status, `${pattern}: ${result.stderr}`).toBe(0);
- }
- const hidden = await fixture.report("hidden-baseline.xml", [
- testcase("hidden"),
- ]);
- expect(compare(hidden, join(directory, ".*.xml")).status).toBe(0);
- for (const name of ["{a,b}.xml", "!report.xml", "@(shard).xml"]) {
- const path = await fixture.report(name, cases);
- expect(compare(baseline, path).status, name).toBe(0);
+ const missing = compare(...args);
+ expect(missing.status).toBe(1);
+ expect(missing.stderr).toContain(absent);
}
});
+ test("reports available baseline timings when the candidate group is empty", async () => {
+ const fixture = await fixtures();
+ const report = await fixture.report("shard.xml", [testcase("present")]);
+ const noBaseline = compare("--", report);
+ expect(noBaseline.status).toBe(1);
+ expect(noBaseline.stderr).toContain("No baseline reports supplied");
+
+ const noCandidate = compare(report, "--");
+ expect(noCandidate.status).toBe(1);
+ expect(noCandidate.stderr).toContain("No candidate reports supplied");
+ expect(noCandidate.stdout).toContain("| shard.xml | 1 | 0 | 1.25 |");
+ });
+
test("detects error status and nested summary failures with failure overriding skipped", async () => {
const fixture = await fixtures();
const baseline = await fixture.report("baseline.xml", [
@@ -434,13 +430,16 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
const fixture = await fixtures();
const first = testcase("first");
const repeated = testcase("parameterized name");
- await fixture.report("baseline-1.xml", [first, repeated]);
- await fixture.report("baseline-2.xml", [repeated]);
- await fixture.report("candidate-1.xml", [repeated, repeated]);
- await fixture.report("candidate-2.xml", [first]);
- const baseline = join(fixture.root, "baseline-*.xml");
- const candidate = join(fixture.root, "candidate-*.xml");
- const result = compare(baseline, candidate);
+ const baseline = [
+ await fixture.report("baseline-1.xml", [first, repeated]),
+ await fixture.report("baseline-2.xml", [repeated]),
+ ];
+ const firstCandidate = await fixture.report("candidate-1.xml", [
+ repeated,
+ repeated,
+ ]);
+ const secondCandidate = await fixture.report("candidate-2.xml", [first]);
+ const result = compare(...baseline, "--", firstCandidate, secondCandidate);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(
"Baseline shards: slowest 1.25s; combined test time: 2.50s.",
@@ -449,33 +448,38 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
"Slowest candidate: 1.25s; combined test time: 2.50s.",
);
const extra = await fixture.report("extra.xml", [repeated]);
- const duplicated = compare(baseline, candidate, extra);
+ const duplicated = compare(
+ ...baseline,
+ "--",
+ firstCandidate,
+ secondCandidate,
+ extra,
+ );
expect(duplicated.status).toBe(1);
expect(duplicated.stderr).toContain("Extra (1)");
- const missing = compare(baseline, join(fixture.root, "candidate-1.xml"));
+ const missing = compare(...baseline, "--", firstCandidate);
expect(missing.status).toBe(1);
expect(missing.stderr).toContain("Missing (1)");
- expect(compare(join(fixture.root, "absent-*.xml"), candidate).status).toBe(
- 1,
- );
});
test("preserves assertion counts for repeated identities across different shard layouts", async () => {
const fixture = await fixtures();
const repeated = (assertions: string) =>
testcase("same parameterized name", "", assertions);
- await fixture.report("baseline-1.xml", [repeated("2"), repeated("5")]);
- await fixture.report("baseline-2.xml", [repeated("5")]);
- await fixture.report("candidate-1.xml", [repeated("5")]);
- await fixture.report("candidate-2.xml", [repeated("2")]);
- await fixture.report("candidate-3.xml", [repeated("05")]);
- const baseline = join(fixture.root, "baseline-*.xml");
- const candidate = join(fixture.root, "candidate-*.xml");
- const matching = compare(baseline, candidate);
+ const baseline = [
+ await fixture.report("baseline-1.xml", [repeated("2"), repeated("5")]),
+ await fixture.report("baseline-2.xml", [repeated("5")]),
+ ];
+ const candidate = [
+ await fixture.report("candidate-1.xml", [repeated("5")]),
+ await fixture.report("candidate-2.xml", [repeated("2")]),
+ await fixture.report("candidate-3.xml", [repeated("05")]),
+ ];
+ const matching = compare(...baseline, "--", ...candidate);
expect(matching.status, matching.stderr).toBe(0);
await fixture.report("candidate-1.xml", [repeated("4")]);
- const changed = compare(baseline, candidate);
+ const changed = compare(...baseline, "--", ...candidate);
expect(changed.status).toBe(1);
expect(changed.stderr).toContain(
"Missing (1): tests-ts/example.test.ts > example > same parameterized name > passed > assertions=5",
@@ -542,8 +546,5 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
);
expect(compare(baseline, candidate).status, name).toBe(1);
}
- expect(compare(baseline, join(fixture.root, "absent-*.xml")).status).toBe(
- 1,
- );
});
});