Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(ci): reuse the JUnit comparison across report fixtures (#1987)

Michael D'Angelodc3516de86d8
diff --git a/sdk/typescript/scripts/compare-test-reports.mts b/sdk/typescript/scripts/compare-test-reports.mts
index e75a36c03..08585b047 100644
--- a/sdk/typescript/scripts/compare-test-reports.mts
+++ b/sdk/typescript/scripts/compare-test-reports.mts
@@ -1,256 +1,3 @@
-// Compare Bun JUnit inventories before changing the required CI runner.
-import { lstatSync, readFileSync, readdirSync, statSync } from "node:fs";
-import { basename, dirname, join, sep } from "node:path";
-import { parseArgs } from "node:util";
-import { minimatch } from "minimatch";
-import { SaxesParser, type SaxesTagNS } from "saxes";
+import { compareTestReports } from "./test-report-comparison.mts";
-type TestStatus = "passed" | "skipped" | "failed";
-type TestIdentity = [file: string, classname: string, name: string];
-type TestOutcome = [
- ...TestIdentity,
- status: TestStatus,
- assertions: string | null,
-];
-type TestRecord = {
- attributes: SaxesTagNS["attributes"];
- status: TestStatus;
-};
-function matchingReports(pattern: string): string[] {
- // Python glob treats ** as one component and does not expand braces/extglobs.
- const directoriesOnly =
- pattern.endsWith(sep) ||
- (process.platform === "win32" && pattern.endsWith("/"));
- if (!/[*?[]/u.test(pattern)) {
- try {
- const stat = directoriesOnly ? statSync(pattern) : lstatSync(pattern);
- return !directoriesOnly || stat.isDirectory() ? [pattern] : [];
- } catch {
- return [];
- }
- }
- const parent = dirname(pattern);
- const namePattern = basename(pattern);
- return (/[*?[]/u.test(parent) ? matchingReports(parent) : [parent]).flatMap(
- (directory) => {
- let names: string[];
- try {
- names = readdirSync(directory);
- } catch {
- return [];
- }
- return names
- .filter(
- (name) =>
- (!name.startsWith(".") || namePattern.startsWith(".")) &&
- minimatch(
- name,
- namePattern.replaceAll("\\", "\\\\").replaceAll("[^", "[\\^"),
- {
- dot: true,
- nobrace: true,
- noext: true,
- noglobstar: true,
- nonegate: true,
- nocomment: true,
- nocase: process.platform === "win32",
- },
- ),
- )
- .map((name) => join(directory, name))
- .filter((path) => {
- if (!directoriesOnly) return true;
- try {
- return statSync(path).isDirectory();
- } catch {
- return false;
- }
- });
- },
- );
-}
-
-function integer(value: string): bigint {
- if (!/^[+-]?\d(?:_?\d)*$/u.test(value.trim())) {
- throw new Error(`invalid integer: ${value}`);
- }
- return BigInt(value.replaceAll("_", "").trim());
-}
-
-function seconds(value: string): number {
- const number = value.trim().replaceAll(/(?<=\d)_(?=\d)/gu, "");
- if (
- !/^[+-]?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|inf(?:inity)?|nan)$/iu.test(
- number,
- )
- ) {
- throw new Error(`invalid duration: ${value}`);
- }
- return Number(number.replace(/inf(?:inity)?/iu, "Infinity"));
-}
-
-function readReport(path: string) {
- const parser = new SaxesParser({ xmlns: true, fileName: path });
- let root: SaxesTagNS | undefined;
- const stack: Array<TestRecord | undefined> = [];
- const records: TestRecord[] = [];
- const suites: SaxesTagNS[] = [];
- parser.on("opentag", (node) => {
- root ??= node;
- const name = node.uri ? `{${node.uri}}${node.local}` : node.local;
- const parent = stack.at(-1);
- if (parent) {
- if (name === "failure" || name === "error") parent.status = "failed";
- else if (name === "skipped" && parent.status === "passed")
- parent.status = "skipped";
- }
- let record: TestRecord | undefined;
- if (name === "testcase") {
- record = { attributes: node.attributes, status: "passed" };
- records.push(record);
- }
- if (name === "testsuite" || name === "testsuites") suites.push(node);
- stack.push(record);
- });
- parser.on("closetag", () => stack.pop());
- const content = new TextDecoder("utf-8", { fatal: true }).decode(
- readFileSync(path),
- );
- parser.write(content).close();
-
- const cases = new Map<string, number>();
- for (const { attributes, status } of records) {
- const identity: TestIdentity = [
- (attributes["file"]?.value ?? "")
- .replaceAll("\\", "/")
- .replace(/^\.\//u, ""),
- attributes["classname"]?.value ?? "",
- attributes["name"]?.value ?? "",
- ];
- const reportedAssertions = attributes["assertions"]?.value;
- const assertions =
- reportedAssertions === undefined
- ? null
- : integer(reportedAssertions).toString();
- const key = JSON.stringify([...identity, status, assertions]);
- cases.set(key, (cases.get(key) ?? 0) + 1);
- }
- if (!cases.size) throw new Error(`${path}: no test cases`);
- if (
- integer(root!.attributes["tests"]?.value ?? String(records.length)) !==
- BigInt(records.length)
- ) {
- throw new Error(`${path}: reported test count does not match test cases`);
- }
- const failed =
- records.some(({ status }) => status === "failed") ||
- suites.some((node) =>
- ["failures", "errors"].some(
- (field) => integer(node.attributes[field]?.value ?? "0") !== 0n,
- ),
- );
- if (failed) console.error(`${path}: test run failed`);
- const duration = seconds(root!.attributes["time"]?.value ?? "0");
- const skipped = records.filter(({ status }) => status === "skipped").length;
- console.log(
- `| ${basename(path)} | ${records.length} | ${skipped} | ${duration.toFixed(2)} |`,
- );
- return {
- cases,
- duration,
- failed,
- };
-}
-
-function readReports(patterns: string[]) {
- const cases = new Map<string, number>();
- const durations: number[] = [];
- let failed = false;
- for (const pattern of patterns) {
- const paths = matchingReports(pattern).sort((a, b) =>
- Buffer.compare(Buffer.from(a), Buffer.from(b)),
- );
- if (!paths.length) throw new Error(`No reports match ${pattern}`);
- for (const path of paths) {
- const report = readReport(path);
- failed ||= report.failed;
- for (const [identity, count] of report.cases)
- cases.set(identity, (cases.get(identity) ?? 0) + count);
- durations.push(report.duration);
- }
- }
- return { cases, durations, failed };
-}
-
-function main(): number {
- let args: { values: { help?: boolean }; positionals: string[] };
- try {
- args = parseArgs({
- options: { help: { type: "boolean", short: "h" } },
- allowPositionals: true,
- });
- if (!args.values.help && args.positionals.length < 2)
- throw new Error(
- "a baseline and at least one candidate report are required",
- );
- } catch (error) {
- console.error((error as Error).message);
- return 2;
- }
- if (args.values.help) {
- console.log(
- "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nBaseline and candidates are JUnit files or glob patterns.",
- );
- return 0;
- }
- console.log("| Report | Cases | Skipped | Seconds |");
- console.log("| --- | ---: | ---: | ---: |");
- const baseline = readReports([args.positionals[0]!]);
- const candidate = readReports(args.positionals.slice(1));
- let failed = baseline.failed || candidate.failed;
- for (const [label, left, right] of [
- ["Missing", baseline.cases, candidate.cases],
- ["Extra", candidate.cases, baseline.cases],
- ] as const) {
- const differences = [...left]
- .map(([identity, count]): [TestOutcome, number] => [
- JSON.parse(identity) as TestOutcome,
- count - (right.get(identity) ?? 0),
- ])
- .filter(([, count]) => count > 0)
- .sort(([a], [b]) => {
- for (let index = 0; index < a.length; index++) {
- const order = Buffer.compare(
- Buffer.from(a[index] ?? ""),
- Buffer.from(b[index] ?? ""),
- );
- if (order) return order;
- }
- return 0;
- });
- for (const [identity, count] of differences) {
- failed = true;
- const [file, classname, name, status, assertions] = identity;
- const description = [file, classname, name, status];
- if (assertions !== null) description.push(`assertions=${assertions}`);
- console.error(`${label} (${count}): ${description.join(" > ")}`);
- }
- }
- if (failed) return 1;
- if (baseline.durations.length > 1) {
- console.log(
- `\nBaseline shards: slowest ${Math.max(...baseline.durations).toFixed(2)}s; combined test time: ${baseline.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.`,
- );
- }
- console.log(
- `\nIdentical test inventory and outcomes. Slowest candidate: ${Math.max(...candidate.durations).toFixed(2)}s; combined test time: ${candidate.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.\n`,
- );
- return 0;
-}
-
-try {
- process.exitCode = main();
-} catch (error) {
- console.error((error as Error).message);
- process.exitCode = 1;
-}
+process.exitCode = compareTestReports(process.argv.slice(2));
diff --git a/sdk/typescript/scripts/test-report-comparison.mts b/sdk/typescript/scripts/test-report-comparison.mts
new file mode 100644
index 000000000..3dbf45769
--- /dev/null
+++ b/sdk/typescript/scripts/test-report-comparison.mts
@@ -0,0 +1,263 @@
+// Compare Bun JUnit inventories before changing the required CI runner.
+import { lstatSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { basename, dirname, join, sep } from "node:path";
+import { parseArgs } from "node:util";
+import { minimatch } from "minimatch";
+import { SaxesParser, type SaxesTagNS } from "saxes";
+
+type Output = Pick<Console, "log" | "error">;
+type TestStatus = "passed" | "skipped" | "failed";
+type TestIdentity = [file: string, classname: string, name: string];
+type TestOutcome = [
+ ...TestIdentity,
+ status: TestStatus,
+ assertions: string | null,
+];
+type TestRecord = {
+ attributes: SaxesTagNS["attributes"];
+ status: TestStatus;
+};
+function matchingReports(pattern: string): string[] {
+ // Python glob treats ** as one component and does not expand braces/extglobs.
+ const directoriesOnly =
+ pattern.endsWith(sep) ||
+ (process.platform === "win32" && pattern.endsWith("/"));
+ if (!/[*?[]/u.test(pattern)) {
+ try {
+ const stat = directoriesOnly ? statSync(pattern) : lstatSync(pattern);
+ return !directoriesOnly || stat.isDirectory() ? [pattern] : [];
+ } catch {
+ return [];
+ }
+ }
+ const parent = dirname(pattern);
+ const namePattern = basename(pattern);
+ return (/[*?[]/u.test(parent) ? matchingReports(parent) : [parent]).flatMap(
+ (directory) => {
+ let names: string[];
+ try {
+ names = readdirSync(directory);
+ } catch {
+ return [];
+ }
+ return names
+ .filter(
+ (name) =>
+ (!name.startsWith(".") || namePattern.startsWith(".")) &&
+ minimatch(
+ name,
+ namePattern.replaceAll("\\", "\\\\").replaceAll("[^", "[\\^"),
+ {
+ dot: true,
+ nobrace: true,
+ noext: true,
+ noglobstar: true,
+ nonegate: true,
+ nocomment: true,
+ nocase: process.platform === "win32",
+ },
+ ),
+ )
+ .map((name) => join(directory, name))
+ .filter((path) => {
+ if (!directoriesOnly) return true;
+ try {
+ return statSync(path).isDirectory();
+ } catch {
+ return false;
+ }
+ });
+ },
+ );
+}
+
+function integer(value: string): bigint {
+ if (!/^[+-]?\d(?:_?\d)*$/u.test(value.trim())) {
+ throw new Error(`invalid integer: ${value}`);
+ }
+ return BigInt(value.replaceAll("_", "").trim());
+}
+
+function seconds(value: string): number {
+ const number = value.trim().replaceAll(/(?<=\d)_(?=\d)/gu, "");
+ if (
+ !/^[+-]?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|inf(?:inity)?|nan)$/iu.test(
+ number,
+ )
+ ) {
+ throw new Error(`invalid duration: ${value}`);
+ }
+ return Number(number.replace(/inf(?:inity)?/iu, "Infinity"));
+}
+
+function readReport(path: string, output: Output) {
+ const parser = new SaxesParser({ xmlns: true, fileName: path });
+ let root: SaxesTagNS | undefined;
+ const stack: Array<TestRecord | undefined> = [];
+ const records: TestRecord[] = [];
+ const suites: SaxesTagNS[] = [];
+ parser.on("opentag", (node) => {
+ root ??= node;
+ const name = node.uri ? `{${node.uri}}${node.local}` : node.local;
+ const parent = stack.at(-1);
+ if (parent) {
+ if (name === "failure" || name === "error") parent.status = "failed";
+ else if (name === "skipped" && parent.status === "passed")
+ parent.status = "skipped";
+ }
+ let record: TestRecord | undefined;
+ if (name === "testcase") {
+ record = { attributes: node.attributes, status: "passed" };
+ records.push(record);
+ }
+ if (name === "testsuite" || name === "testsuites") suites.push(node);
+ stack.push(record);
+ });
+ parser.on("closetag", () => stack.pop());
+ const content = new TextDecoder("utf-8", { fatal: true }).decode(
+ readFileSync(path),
+ );
+ parser.write(content).close();
+
+ const cases = new Map<string, number>();
+ for (const { attributes, status } of records) {
+ const identity: TestIdentity = [
+ (attributes["file"]?.value ?? "")
+ .replaceAll("\\", "/")
+ .replace(/^\.\//u, ""),
+ attributes["classname"]?.value ?? "",
+ attributes["name"]?.value ?? "",
+ ];
+ const reportedAssertions = attributes["assertions"]?.value;
+ const assertions =
+ reportedAssertions === undefined
+ ? null
+ : integer(reportedAssertions).toString();
+ const key = JSON.stringify([...identity, status, assertions]);
+ cases.set(key, (cases.get(key) ?? 0) + 1);
+ }
+ if (!cases.size) throw new Error(`${path}: no test cases`);
+ if (
+ integer(root!.attributes["tests"]?.value ?? String(records.length)) !==
+ BigInt(records.length)
+ ) {
+ throw new Error(`${path}: reported test count does not match test cases`);
+ }
+ const failed =
+ records.some(({ status }) => status === "failed") ||
+ suites.some((node) =>
+ ["failures", "errors"].some(
+ (field) => integer(node.attributes[field]?.value ?? "0") !== 0n,
+ ),
+ );
+ if (failed) output.error(`${path}: test run failed`);
+ const duration = seconds(root!.attributes["time"]?.value ?? "0");
+ const skipped = records.filter(({ status }) => status === "skipped").length;
+ output.log(
+ `| ${basename(path)} | ${records.length} | ${skipped} | ${duration.toFixed(2)} |`,
+ );
+ return {
+ cases,
+ duration,
+ failed,
+ };
+}
+
+function readReports(patterns: string[], output: Output) {
+ const cases = new Map<string, number>();
+ const durations: number[] = [];
+ let failed = false;
+ for (const pattern of patterns) {
+ const paths = matchingReports(pattern).sort((a, b) =>
+ Buffer.compare(Buffer.from(a), Buffer.from(b)),
+ );
+ if (!paths.length) throw new Error(`No reports match ${pattern}`);
+ for (const path of paths) {
+ const report = readReport(path, output);
+ failed ||= report.failed;
+ for (const [identity, count] of report.cases)
+ cases.set(identity, (cases.get(identity) ?? 0) + count);
+ durations.push(report.duration);
+ }
+ }
+ return { cases, durations, failed };
+}
+
+function compareGroups(patterns: string[], output: Output): number {
+ output.log("| Report | Cases | Skipped | Seconds |");
+ output.log("| --- | ---: | ---: | ---: |");
+ const baseline = readReports([patterns[0]!], output);
+ const candidate = readReports(patterns.slice(1), output);
+ let failed = baseline.failed || candidate.failed;
+ for (const [label, left, right] of [
+ ["Missing", baseline.cases, candidate.cases],
+ ["Extra", candidate.cases, baseline.cases],
+ ] as const) {
+ const differences = [...left]
+ .map(([identity, count]): [TestOutcome, number] => [
+ JSON.parse(identity) as TestOutcome,
+ count - (right.get(identity) ?? 0),
+ ])
+ .filter(([, count]) => count > 0)
+ .sort(([a], [b]) => {
+ for (let index = 0; index < a.length; index++) {
+ const order = Buffer.compare(
+ Buffer.from(a[index] ?? ""),
+ Buffer.from(b[index] ?? ""),
+ );
+ if (order) return order;
+ }
+ return 0;
+ });
+ for (const [identity, count] of differences) {
+ failed = true;
+ const [file, classname, name, status, assertions] = identity;
+ const description = [file, classname, name, status];
+ if (assertions !== null) description.push(`assertions=${assertions}`);
+ output.error(`${label} (${count}): ${description.join(" > ")}`);
+ }
+ }
+ if (failed) return 1;
+ if (baseline.durations.length > 1) {
+ output.log(
+ `\nBaseline shards: slowest ${Math.max(...baseline.durations).toFixed(2)}s; combined test time: ${baseline.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.`,
+ );
+ }
+ output.log(
+ `\nIdentical test inventory and outcomes. Slowest candidate: ${Math.max(...candidate.durations).toFixed(2)}s; combined test time: ${candidate.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.\n`,
+ );
+ return 0;
+}
+
+export function compareTestReports(
+ argv: string[],
+ output: Output = console,
+): number {
+ let args: { values: { help?: boolean }; positionals: string[] };
+ try {
+ args = parseArgs({
+ args: argv,
+ options: { help: { type: "boolean", short: "h" } },
+ allowPositionals: true,
+ });
+ if (!args.values.help && args.positionals.length < 2)
+ throw new Error(
+ "a baseline and at least one candidate report are required",
+ );
+ } catch (error) {
+ output.error((error as Error).message);
+ return 2;
+ }
+ if (args.values.help) {
+ output.log(
+ "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nBaseline and candidates are JUnit files or glob patterns.",
+ );
+ return 0;
+ }
+ try {
+ return compareGroups(args.positionals, output);
+ } catch (error) {
+ output.error((error as Error).message);
+ return 1;
+ }
+}
diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts
index e03386c24..e7a5c7ce7 100644
--- a/sdk/typescript/tests-ts/test-reports.test.ts
+++ b/sdk/typescript/tests-ts/test-reports.test.ts
@@ -1,7 +1,8 @@
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { afterEach, describe, expect, test } from "bun:test";
-import { bashCommand, readSubprocess } from "./support/shell.js";
+import { compareTestReports } from "../scripts/test-report-comparison.mts";
+import { bashCommand, runCommand } from "./support/shell.js";
import { createTemporaryDirectories } from "./support/temporary-directories.js";
// Node 22.13 emits this upstream notice when executing TypeScript directly.
@@ -9,6 +10,12 @@ const typeStrippingNotice =
/^(?:\(node:\d+\) ExperimentalWarning: Type Stripping is an experimental feature and might change at any time\n\(Use `node --trace-warnings \.\.\.` to show where the warning was created\)\n)?$/u;
const bash = bashCommand();
+const comparator = join(
+ import.meta.dirname,
+ "..",
+ "scripts",
+ "compare-test-reports.mts",
+);
const directories = createTemporaryDirectories(false);
afterEach(directories.cleanup);
@@ -58,18 +65,20 @@ async function workflowFixtures() {
};
await mkdir(join(fixture.root, "reports"));
const { target } = workflow.jobs.runner.strategy.matrix;
- for (const { os, shard, count, mode } of target) {
- const totalCases = Math.max(
- ...target.filter((item) => item.os === os).map((item) => item.count),
- );
- const casesPerShard = totalCases / count;
- await fixture.report(
- `reports/runner-${os}-${mode}-${shard}.xml`,
- Array.from({ length: casesPerShard }, (_, index) =>
- testcase(`${os} case ${(shard - 1) * casesPerShard + index + 1}`),
- ),
- );
- }
+ await Promise.all(
+ target.map(({ os, shard, count, mode }) => {
+ const totalCases = Math.max(
+ ...target.filter((item) => item.os === os).map((item) => item.count),
+ );
+ const casesPerShard = totalCases / count;
+ return fixture.report(
+ `reports/runner-${os}-${mode}-${shard}.xml`,
+ Array.from({ length: casesPerShard }, (_, index) =>
+ testcase(`${os} case ${(shard - 1) * casesPerShard + index + 1}`),
+ ),
+ );
+ }),
+ );
return {
...fixture,
script: workflow.jobs.compare.steps.find(
@@ -84,26 +93,49 @@ function testcase(name: string, status = "", assertions?: string) {
return `<testcase file="tests-ts/example.test.ts" classname="example" name="${name}"${assertionAttribute}>${status}</testcase>`;
}
-async function compare(baseline: string, ...candidates: string[]) {
- const child = Bun.spawn({
- cmd: [
- "node",
- "--experimental-strip-types",
- join(import.meta.dirname, "..", "scripts", "compare-test-reports.mts"),
- baseline,
- ...candidates,
- ],
- stdin: "ignore",
- stdout: "pipe",
- stderr: "pipe",
+function compare(...paths: string[]) {
+ let stdout = "",
+ stderr = "";
+ const status = compareTestReports(paths, {
+ log: (line) => {
+ stdout += line + "\n";
+ },
+ error: (line) => {
+ stderr += line + "\n";
+ },
});
- return readSubprocess(child);
+ return { status, stdout, stderr };
+}
+
+function compareCli(...paths: string[]) {
+ return runCommand("node", [
+ "--experimental-strip-types",
+ comparator,
+ ...paths,
+ ]);
+}
+
+async function runWorkflow(
+ fixture: Awaited<ReturnType<typeof workflowFixtures>>,
+ node: string,
+ env?: NodeJS.ProcessEnv,
+) {
+ const summary = join(fixture.root, "summary.md");
+ await writeFile(summary, "");
+ const result = await runCommand(
+ bash,
+ ["-e", "-o", "pipefail", "-c", node + "\n" + fixture.script],
+ {
+ cwd: fixture.root,
+ env: { ...process.env, GITHUB_STEP_SUMMARY: "summary.md", ...env },
+ },
+ );
+ return { ...result, summary: await readFile(summary, "utf8") };
}
describe("JUnit inventory comparison", () => {
test("runs every workflow comparison before reporting a mismatch", async () => {
const fixture = await workflowFixtures();
- const script = fixture.script;
const expected = ["ubuntu-latest", "macos-latest", "windows-latest"].map(
(os) =>
`reports/runner-${os}-baseline-*.xml|reports/runner-${os}-expanded-*.xml`,
@@ -112,27 +144,12 @@ describe("JUnit inventory comparison", () => {
printf '%s\\n' "$3|$4"
[[ "$3|$4" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]
}`;
- const summary = join(fixture.root, "summary.md");
for (const failedReport of ["", expected[0]!]) {
- await writeFile(summary, "");
- const child = Bun.spawn({
- cmd: [bash, "-e", "-o", "pipefail", "-c", `${mock}\n${script}`],
- cwd: fixture.root,
- stdin: "ignore",
- stdout: "ignore",
- stderr: "pipe",
- env: {
- ...process.env,
- GITHUB_STEP_SUMMARY: "summary.md",
- CODEX_SECURITY_TEST_FAIL_REPORT: failedReport,
- },
- timeout: 10_000,
+ const { status, stderr, summary } = await runWorkflow(fixture, mock, {
+ CODEX_SECURITY_TEST_FAIL_REPORT: failedReport,
});
- const { status, stderr } = await readSubprocess(child);
expect(status, stderr).toBe(failedReport === "" ? 0 : 1);
- expect((await readFile(summary, "utf8")).trim().split(/\r?\n/u)).toEqual(
- expected,
- );
+ expect(summary.trim().split(/\r?\n/u)).toEqual(expected);
}
});
@@ -147,35 +164,12 @@ describe("JUnit inventory comparison", () => {
);
}
}
- await writeFile(join(fixture.root, "summary.md"), "");
- const child = Bun.spawn({
- cmd: [
- bash,
- "-e",
- "-o",
- "pipefail",
- "-c",
- `node() { command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$3" "$4"; }\n${fixture.script}`,
- ],
- cwd: fixture.root,
- stdin: "ignore",
- stdout: "ignore",
- stderr: "pipe",
- env: {
- ...process.env,
- GITHUB_STEP_SUMMARY: "summary.md",
- CODEX_SECURITY_TEST_COMPARATOR: join(
- import.meta.dirname,
- "..",
- "scripts",
- "compare-test-reports.mts",
- ),
- },
- timeout: 10_000,
- });
- const { status, stderr } = await readSubprocess(child);
+ const { status, stderr, summary } = await runWorkflow(
+ fixture,
+ 'node() { command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$3" "$4"; }',
+ { CODEX_SECURITY_TEST_COMPARATOR: comparator },
+ );
expect(status, stderr).toBe(incomplete ? 1 : 0);
- const summary = await readFile(join(fixture.root, "summary.md"), "utf8");
expect(
summary.match(/Identical test inventory and outcomes/gu),
).toHaveLength(3);
@@ -188,115 +182,61 @@ describe("JUnit inventory comparison", () => {
}
});
- for (const scenario of [
- {
- name: "missing baseline Linux shard",
- os: "ubuntu-latest",
- mode: "baseline",
- count: 3,
- missing: 3,
- },
- {
- name: "missing expanded Linux shard",
- os: "ubuntu-latest",
- mode: "expanded",
- count: 6,
- missing: 6,
- },
- {
- name: "missing baseline Windows shard",
- os: "windows-latest",
- mode: "baseline",
- count: 7,
- missing: 7,
- },
- {
- name: "missing expanded Windows shard",
- os: "windows-latest",
- mode: "expanded",
- count: 14,
- missing: 14,
- },
- {
- name: "missing baseline shard replaced by an unexpected index",
- os: "ubuntu-latest",
- mode: "baseline",
- count: 3,
- missing: 3,
- extra: 4,
- },
- {
- name: "missing expanded shard replaced by an unexpected index",
- os: "ubuntu-latest",
- mode: "expanded",
- count: 6,
- missing: 6,
- extra: 7,
- },
- {
- name: "extra baseline shard",
- os: "windows-latest",
- mode: "baseline",
- count: 7,
- extra: 8,
- },
- {
- name: "extra expanded shard",
- os: "windows-latest",
- mode: "expanded",
- count: 14,
- extra: 15,
- },
- ]) {
- test(`rejects incomplete workflow groups: ${scenario.name}`, async () => {
+ for (const [name, os, mode, count, issue] of [
+ ["missing baseline Linux shard", "ubuntu-latest", "baseline", 3, "missing"],
+ ["missing expanded Linux shard", "ubuntu-latest", "expanded", 6, "missing"],
+ [
+ "missing baseline Windows shard",
+ "windows-latest",
+ "baseline",
+ 7,
+ "missing",
+ ],
+ [
+ "missing expanded Windows shard",
+ "windows-latest",
+ "expanded",
+ 14,
+ "missing",
+ ],
+ [
+ "missing baseline shard replaced by an unexpected index",
+ "ubuntu-latest",
+ "baseline",
+ 3,
+ "replaced",
+ ],
+ [
+ "missing expanded shard replaced by an unexpected index",
+ "ubuntu-latest",
+ "expanded",
+ 6,
+ "replaced",
+ ],
+ ["extra baseline shard", "windows-latest", "baseline", 7, "extra"],
+ ["extra expanded shard", "windows-latest", "expanded", 14, "extra"],
+ ] as const) {
+ test(`rejects incomplete workflow groups: ${name}`, async () => {
const fixture = await workflowFixtures();
- if (scenario.missing !== undefined) {
- await rm(
- join(
- fixture.root,
- `reports/runner-${scenario.os}-${scenario.mode}-${scenario.missing}.xml`,
- ),
- );
- }
- if (scenario.extra !== undefined) {
- await fixture.report(
- `reports/runner-${scenario.os}-${scenario.mode}-${scenario.extra}.xml`,
- [testcase("unexpected shard")],
- );
- }
- const child = Bun.spawn({
- cmd: [
- bash,
- "-e",
- "-o",
- "pipefail",
- "-c",
- `node() { printf '%s\\n' "$3|$4"; }\n${fixture.script}`,
- ],
- cwd: fixture.root,
- stdin: "ignore",
- stdout: "ignore",
- stderr: "pipe",
- env: { ...process.env, GITHUB_STEP_SUMMARY: "summary.md" },
- timeout: 10_000,
- });
- const { status, stderr } = await readSubprocess(child);
+ const prefix = `reports/runner-${os}-${mode}-`;
+ if (issue !== "extra")
+ await rm(join(fixture.root, `${prefix}${count}.xml`));
+ if (issue !== "missing")
+ await fixture.report(`${prefix}${count + 1}.xml`, [
+ testcase("unexpected shard"),
+ ]);
+ const { status, stderr, summary } = await runWorkflow(
+ fixture,
+ `node() { printf '%s\\n' "$3|$4"; }`,
+ );
expect(status, stderr).toBe(1);
- if (scenario.missing !== undefined) {
- expect(stderr).toContain(
- `Missing shard report: reports/runner-${scenario.os}-${scenario.mode}-${scenario.missing}.xml`,
- );
- } else {
- expect(stderr).toContain(
- `Expected ${scenario.count} reports for ${scenario.os}/${scenario.mode}, found ${scenario.count + 1}`,
- );
- }
+ expect(stderr).toContain(
+ issue !== "extra"
+ ? `Missing shard report: ${prefix}${count}.xml`
+ : `Expected ${count} reports for ${os}/${mode}, found ${count + 1}`,
+ );
// Missing artifacts must not prevent the other OS/mode diagnostics.
- expect(
- (await readFile(join(fixture.root, "summary.md"), "utf8"))
- .trim()
- .split(/\r?\n/u),
- ).toHaveLength(3);
+ expect(summary.trim().split(/\r?\n/u)).toHaveLength(3);
});
}
@@ -307,7 +247,10 @@ describe("JUnit inventory comparison", () => {
const baseline = await fixture.report("baseline.xml", [passed, skipped]);
await fixture.report("shard-1.xml", [skipped]);
await fixture.report("shard-2.xml", [passed]);
- const result = await compare(baseline, join(fixture.root, "shard-*.xml"));
+ const result = await compareCli(
+ baseline,
+ join(fixture.root, "shard-*.xml"),
+ );
expect(result.status, result.stderr).toBe(0);
expect(result.stderr).toMatch(typeStrippingNotice);
expect(result.stdout).toBe(
@@ -335,7 +278,7 @@ describe("JUnit inventory comparison", () => {
${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
</testsuite></testsuites>`,
);
- const result = await compare(baseline, candidate);
+ const result = compare(baseline, candidate);
expect(result.status, result.stderr).toBe(0);
expect(result.stderr).toMatch(typeStrippingNotice);
});
@@ -350,8 +293,8 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
"./tests-ts\\example.test.ts",
),
]);
- expect((await compare(baseline, candidate)).status).toBe(0);
- const result = await compare(baseline, candidate, candidate);
+ expect(compare(baseline, candidate).status).toBe(0);
+ const result = await compareCli(baseline, candidate, candidate);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"Extra (1): tests-ts/example.test.ts > example > portable > passed",
@@ -379,16 +322,16 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
"reports with spaces/*.xml",
"**/shard-*.xml",
]) {
- const result = await compare(baseline, join(fixture.root, pattern));
+ const result = compare(baseline, join(fixture.root, pattern));
expect(result.status, `${pattern}: ${result.stderr}`).toBe(0);
}
const hidden = await fixture.report("hidden-baseline.xml", [
testcase("hidden"),
]);
- expect((await compare(hidden, join(directory, ".*.xml"))).status).toBe(0);
+ expect(compare(hidden, join(directory, ".*.xml")).status).toBe(0);
for (const name of ["{a,b}.xml", "!report.xml", "@(shard).xml"]) {
const path = await fixture.report(name, cases);
- expect((await compare(baseline, path)).status, name).toBe(0);
+ expect(compare(baseline, path).status, name).toBe(0);
}
});
@@ -401,7 +344,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
const candidate = await fixture.report("error.xml", [
testcase("example", status),
]);
- const result = await compare(baseline, candidate);
+ const result = compare(baseline, candidate);
expect(result.status).toBe(1);
expect(result.stderr).toContain("test run failed");
expect(result.stderr).toContain("example > example > failed");
@@ -412,7 +355,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
nested,
`<testsuites tests="1"><testsuite><testsuite errors="1">${testcase("example")}</testsuite></testsuite></testsuites>`,
);
- const result = await compare(baseline, nested);
+ const result = compare(baseline, nested);
expect(result.status).toBe(1);
expect(result.stderr).toContain("test run failed");
});
@@ -434,7 +377,10 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
]) {
const candidate = join(fixture.root, "invalid.xml");
await writeFile(candidate, xml);
- const result = await compare(baseline, candidate);
+ const result =
+ xml === "<testsuite><testcase/></testsuite><extra/>"
+ ? await compareCli(baseline, candidate)
+ : compare(baseline, candidate);
expect(result.status, xml).toBe(1);
expect(result.stdout).not.toContain("Identical test inventory");
}
@@ -450,7 +396,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
candidate,
`<testsuite xmlns:other="urn:example">${testcase("example", "<properties><failure/></properties><other:error/>")}</testsuite>`,
);
- const result = await compare(baseline, candidate);
+ const result = compare(baseline, candidate);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain("| defaults.xml | 1 | 0 | 0.00 |");
});
@@ -469,7 +415,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
passed,
passed,
]);
- const result = await compare(baseline, candidate);
+ const result = compare(baseline, candidate);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain("| baseline.xml | 3 | 1 | 1.25 |");
for (const [name, cases, diagnostic] of [
@@ -478,7 +424,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
["changed", [passed, skipped, skipped], "Missing (1)"],
] as const) {
const changed = await fixture.report(`${name}.xml`, [...cases]);
- const comparison = await compare(baseline, changed);
+ const comparison = compare(baseline, changed);
expect(comparison.status, name).toBe(1);
expect(comparison.stderr).toContain(diagnostic);
}
@@ -494,7 +440,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
await fixture.report("candidate-2.xml", [first]);
const baseline = join(fixture.root, "baseline-*.xml");
const candidate = join(fixture.root, "candidate-*.xml");
- const result = await compare(baseline, candidate);
+ const result = compare(baseline, candidate);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(
"Baseline shards: slowest 1.25s; combined test time: 2.50s.",
@@ -503,18 +449,15 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
"Slowest candidate: 1.25s; combined test time: 2.50s.",
);
const extra = await fixture.report("extra.xml", [repeated]);
- const duplicated = await compare(baseline, candidate, extra);
+ const duplicated = compare(baseline, candidate, extra);
expect(duplicated.status).toBe(1);
expect(duplicated.stderr).toContain("Extra (1)");
- const missing = await compare(
- baseline,
- join(fixture.root, "candidate-1.xml"),
- );
+ const missing = compare(baseline, join(fixture.root, "candidate-1.xml"));
expect(missing.status).toBe(1);
expect(missing.stderr).toContain("Missing (1)");
- expect(
- (await compare(join(fixture.root, "absent-*.xml"), candidate)).status,
- ).toBe(1);
+ expect(compare(join(fixture.root, "absent-*.xml"), candidate).status).toBe(
+ 1,
+ );
});
test("preserves assertion counts for repeated identities across different shard layouts", async () => {
@@ -528,11 +471,11 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
await fixture.report("candidate-3.xml", [repeated("05")]);
const baseline = join(fixture.root, "baseline-*.xml");
const candidate = join(fixture.root, "candidate-*.xml");
- const matching = await compare(baseline, candidate);
+ const matching = compare(baseline, candidate);
expect(matching.status, matching.stderr).toBe(0);
await fixture.report("candidate-1.xml", [repeated("4")]);
- const changed = await compare(baseline, candidate);
+ const changed = compare(baseline, candidate);
expect(changed.status).toBe(1);
expect(changed.stderr).toContain(
"Missing (1): tests-ts/example.test.ts > example > same parameterized name > passed > assertions=5",
@@ -550,7 +493,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
const candidate = await fixture.report("candidate.xml", [
testcase("without assertions"),
]);
- const result = await compare(baseline, candidate);
+ const result = compare(baseline, candidate);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"Missing (1): tests-ts/example.test.ts > example > without assertions > passed > assertions=0",
@@ -569,7 +512,7 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
testcase("first", "<failure/>"),
]);
const passed = await fixture.report("shard-2.xml", [second]);
- const result = await compare(baseline, failed, passed);
+ const result = compare(baseline, failed, passed);
expect(result.status).toBe(1);
expect(result.stderr).toContain("test run failed");
@@ -597,10 +540,10 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
failures,
count,
);
- expect((await compare(baseline, candidate)).status, name).toBe(1);
+ expect(compare(baseline, candidate).status, name).toBe(1);
}
- expect(
- (await compare(baseline, join(fixture.root, "absent-*.xml"))).status,
- ).toBe(1);
+ expect(compare(baseline, join(fixture.root, "absent-*.xml")).status).toBe(
+ 1,
+ );
});
});