OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(ci): reuse the JUnit comparison across report fixtures (#1987)
Michael D'Angelodc3516de86d8
diff --git a/sdk/typescript/scripts/compare-test-reports.mts b/sdk/typescript/scripts/compare-test-reports.mtsindex e75a36c03..08585b047 100644--- a/sdk/typescript/scripts/compare-test-reports.mts+++ b/sdk/typescript/scripts/compare-test-reports.mts@@ -1,256 +1,3 @@-// Compare Bun JUnit inventories before changing the required CI runner.-import { lstatSync, readFileSync, readdirSync, statSync } from "node:fs";-import { basename, dirname, join, sep } from "node:path";-import { parseArgs } from "node:util";-import { minimatch } from "minimatch";-import { SaxesParser, type SaxesTagNS } from "saxes";+import { compareTestReports } from "./test-report-comparison.mts";-type TestStatus = "passed" | "skipped" | "failed";-type TestIdentity = [file: string, classname: string, name: string];-type TestOutcome = [- ...TestIdentity,- status: TestStatus,- assertions: string | null,-];-type TestRecord = {- attributes: SaxesTagNS["attributes"];- status: TestStatus;-};-function matchingReports(pattern: string): string[] {- // Python glob treats ** as one component and does not expand braces/extglobs.- const directoriesOnly =- pattern.endsWith(sep) ||- (process.platform === "win32" && pattern.endsWith("/"));- if (!/[*?[]/u.test(pattern)) {- try {- const stat = directoriesOnly ? statSync(pattern) : lstatSync(pattern);- return !directoriesOnly || stat.isDirectory() ? [pattern] : [];- } catch {- return [];- }- }- const parent = dirname(pattern);- const namePattern = basename(pattern);- return (/[*?[]/u.test(parent) ? matchingReports(parent) : [parent]).flatMap(- (directory) => {- let names: string[];- try {- names = readdirSync(directory);- } catch {- return [];- }- return names- .filter(- (name) =>- (!name.startsWith(".") || namePattern.startsWith(".")) &&- minimatch(- name,- namePattern.replaceAll("\\", "\\\\").replaceAll("[^", "[\\^"),- {- dot: true,- nobrace: true,- noext: true,- noglobstar: true,- nonegate: true,- nocomment: true,- nocase: process.platform === "win32",- },- ),- )- .map((name) => join(directory, name))- .filter((path) => {- if (!directoriesOnly) return true;- try {- return statSync(path).isDirectory();- } catch {- return false;- }- });- },- );-}--function integer(value: string): bigint {- if (!/^[+-]?\d(?:_?\d)*$/u.test(value.trim())) {- throw new Error(`invalid integer: ${value}`);- }- return BigInt(value.replaceAll("_", "").trim());-}--function seconds(value: string): number {- const number = value.trim().replaceAll(/(?<=\d)_(?=\d)/gu, "");- if (- !/^[+-]?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|inf(?:inity)?|nan)$/iu.test(- number,- )- ) {- throw new Error(`invalid duration: ${value}`);- }- return Number(number.replace(/inf(?:inity)?/iu, "Infinity"));-}--function readReport(path: string) {- const parser = new SaxesParser({ xmlns: true, fileName: path });- let root: SaxesTagNS | undefined;- const stack: Array<TestRecord | undefined> = [];- const records: TestRecord[] = [];- const suites: SaxesTagNS[] = [];- parser.on("opentag", (node) => {- root ??= node;- const name = node.uri ? `{${node.uri}}${node.local}` : node.local;- const parent = stack.at(-1);- if (parent) {- if (name === "failure" || name === "error") parent.status = "failed";- else if (name === "skipped" && parent.status === "passed")- parent.status = "skipped";- }- let record: TestRecord | undefined;- if (name === "testcase") {- record = { attributes: node.attributes, status: "passed" };- records.push(record);- }- if (name === "testsuite" || name === "testsuites") suites.push(node);- stack.push(record);- });- parser.on("closetag", () => stack.pop());- const content = new TextDecoder("utf-8", { fatal: true }).decode(- readFileSync(path),- );- parser.write(content).close();-- const cases = new Map<string, number>();- for (const { attributes, status } of records) {- const identity: TestIdentity = [- (attributes["file"]?.value ?? "")- .replaceAll("\\", "/")- .replace(/^\.\//u, ""),- attributes["classname"]?.value ?? "",- attributes["name"]?.value ?? "",- ];- const reportedAssertions = attributes["assertions"]?.value;- const assertions =- reportedAssertions === undefined- ? null- : integer(reportedAssertions).toString();- const key = JSON.stringify([...identity, status, assertions]);- cases.set(key, (cases.get(key) ?? 0) + 1);- }- if (!cases.size) throw new Error(`${path}: no test cases`);- if (- integer(root!.attributes["tests"]?.value ?? String(records.length)) !==- BigInt(records.length)- ) {- throw new Error(`${path}: reported test count does not match test cases`);- }- const failed =- records.some(({ status }) => status === "failed") ||- suites.some((node) =>- ["failures", "errors"].some(- (field) => integer(node.attributes[field]?.value ?? "0") !== 0n,- ),- );- if (failed) console.error(`${path}: test run failed`);- const duration = seconds(root!.attributes["time"]?.value ?? "0");- const skipped = records.filter(({ status }) => status === "skipped").length;- console.log(- `| ${basename(path)} | ${records.length} | ${skipped} | ${duration.toFixed(2)} |`,- );- return {- cases,- duration,- failed,- };-}--function readReports(patterns: string[]) {- const cases = new Map<string, number>();- const durations: number[] = [];- let failed = false;- for (const pattern of patterns) {- const paths = matchingReports(pattern).sort((a, b) =>- Buffer.compare(Buffer.from(a), Buffer.from(b)),- );- if (!paths.length) throw new Error(`No reports match ${pattern}`);- for (const path of paths) {- const report = readReport(path);- failed ||= report.failed;- for (const [identity, count] of report.cases)- cases.set(identity, (cases.get(identity) ?? 0) + count);- durations.push(report.duration);- }- }- return { cases, durations, failed };-}--function main(): number {- let args: { values: { help?: boolean }; positionals: string[] };- try {- args = parseArgs({- options: { help: { type: "boolean", short: "h" } },- allowPositionals: true,- });- if (!args.values.help && args.positionals.length < 2)- throw new Error(- "a baseline and at least one candidate report are required",- );- } catch (error) {- console.error((error as Error).message);- return 2;- }- if (args.values.help) {- console.log(- "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nBaseline and candidates are JUnit files or glob patterns.",- );- return 0;- }- console.log("| Report | Cases | Skipped | Seconds |");- console.log("| --- | ---: | ---: | ---: |");- const baseline = readReports([args.positionals[0]!]);- const candidate = readReports(args.positionals.slice(1));- let failed = baseline.failed || candidate.failed;- for (const [label, left, right] of [- ["Missing", baseline.cases, candidate.cases],- ["Extra", candidate.cases, baseline.cases],- ] as const) {- const differences = [...left]- .map(([identity, count]): [TestOutcome, number] => [- JSON.parse(identity) as TestOutcome,- count - (right.get(identity) ?? 0),- ])- .filter(([, count]) => count > 0)- .sort(([a], [b]) => {- for (let index = 0; index < a.length; index++) {- const order = Buffer.compare(- Buffer.from(a[index] ?? ""),- Buffer.from(b[index] ?? ""),- );- if (order) return order;- }- return 0;- });- for (const [identity, count] of differences) {- failed = true;- const [file, classname, name, status, assertions] = identity;- const description = [file, classname, name, status];- if (assertions !== null) description.push(`assertions=${assertions}`);- console.error(`${label} (${count}): ${description.join(" > ")}`);- }- }- if (failed) return 1;- if (baseline.durations.length > 1) {- console.log(- `\nBaseline shards: slowest ${Math.max(...baseline.durations).toFixed(2)}s; combined test time: ${baseline.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.`,- );- }- console.log(- `\nIdentical test inventory and outcomes. Slowest candidate: ${Math.max(...candidate.durations).toFixed(2)}s; combined test time: ${candidate.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.\n`,- );- return 0;-}--try {- process.exitCode = main();-} catch (error) {- console.error((error as Error).message);- process.exitCode = 1;-}+process.exitCode = compareTestReports(process.argv.slice(2));diff --git a/sdk/typescript/scripts/test-report-comparison.mts b/sdk/typescript/scripts/test-report-comparison.mtsnew file mode 100644index 000000000..3dbf45769--- /dev/null+++ b/sdk/typescript/scripts/test-report-comparison.mts@@ -0,0 +1,263 @@+// Compare Bun JUnit inventories before changing the required CI runner.+import { lstatSync, readFileSync, readdirSync, statSync } from "node:fs";+import { basename, dirname, join, sep } from "node:path";+import { parseArgs } from "node:util";+import { minimatch } from "minimatch";+import { SaxesParser, type SaxesTagNS } from "saxes";++type Output = Pick<Console, "log" | "error">;+type TestStatus = "passed" | "skipped" | "failed";+type TestIdentity = [file: string, classname: string, name: string];+type TestOutcome = [+ ...TestIdentity,+ status: TestStatus,+ assertions: string | null,+];+type TestRecord = {+ attributes: SaxesTagNS["attributes"];+ status: TestStatus;+};+function matchingReports(pattern: string): string[] {+ // Python glob treats ** as one component and does not expand braces/extglobs.+ const directoriesOnly =+ pattern.endsWith(sep) ||+ (process.platform === "win32" && pattern.endsWith("/"));+ if (!/[*?[]/u.test(pattern)) {+ try {+ const stat = directoriesOnly ? statSync(pattern) : lstatSync(pattern);+ return !directoriesOnly || stat.isDirectory() ? [pattern] : [];+ } catch {+ return [];+ }+ }+ const parent = dirname(pattern);+ const namePattern = basename(pattern);+ return (/[*?[]/u.test(parent) ? matchingReports(parent) : [parent]).flatMap(+ (directory) => {+ let names: string[];+ try {+ names = readdirSync(directory);+ } catch {+ return [];+ }+ return names+ .filter(+ (name) =>+ (!name.startsWith(".") || namePattern.startsWith(".")) &&+ minimatch(+ name,+ namePattern.replaceAll("\\", "\\\\").replaceAll("[^", "[\\^"),+ {+ dot: true,+ nobrace: true,+ noext: true,+ noglobstar: true,+ nonegate: true,+ nocomment: true,+ nocase: process.platform === "win32",+ },+ ),+ )+ .map((name) => join(directory, name))+ .filter((path) => {+ if (!directoriesOnly) return true;+ try {+ return statSync(path).isDirectory();+ } catch {+ return false;+ }+ });+ },+ );+}++function integer(value: string): bigint {+ if (!/^[+-]?\d(?:_?\d)*$/u.test(value.trim())) {+ throw new Error(`invalid integer: ${value}`);+ }+ return BigInt(value.replaceAll("_", "").trim());+}++function seconds(value: string): number {+ const number = value.trim().replaceAll(/(?<=\d)_(?=\d)/gu, "");+ if (+ !/^[+-]?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|inf(?:inity)?|nan)$/iu.test(+ number,+ )+ ) {+ throw new Error(`invalid duration: ${value}`);+ }+ return Number(number.replace(/inf(?:inity)?/iu, "Infinity"));+}++function readReport(path: string, output: Output) {+ const parser = new SaxesParser({ xmlns: true, fileName: path });+ let root: SaxesTagNS | undefined;+ const stack: Array<TestRecord | undefined> = [];+ const records: TestRecord[] = [];+ const suites: SaxesTagNS[] = [];+ parser.on("opentag", (node) => {+ root ??= node;+ const name = node.uri ? `{${node.uri}}${node.local}` : node.local;+ const parent = stack.at(-1);+ if (parent) {+ if (name === "failure" || name === "error") parent.status = "failed";+ else if (name === "skipped" && parent.status === "passed")+ parent.status = "skipped";+ }+ let record: TestRecord | undefined;+ if (name === "testcase") {+ record = { attributes: node.attributes, status: "passed" };+ records.push(record);+ }+ if (name === "testsuite" || name === "testsuites") suites.push(node);+ stack.push(record);+ });+ parser.on("closetag", () => stack.pop());+ const content = new TextDecoder("utf-8", { fatal: true }).decode(+ readFileSync(path),+ );+ parser.write(content).close();++ const cases = new Map<string, number>();+ for (const { attributes, status } of records) {+ const identity: TestIdentity = [+ (attributes["file"]?.value ?? "")+ .replaceAll("\\", "/")+ .replace(/^\.\//u, ""),+ attributes["classname"]?.value ?? "",+ attributes["name"]?.value ?? "",+ ];+ const reportedAssertions = attributes["assertions"]?.value;+ const assertions =+ reportedAssertions === undefined+ ? null+ : integer(reportedAssertions).toString();+ const key = JSON.stringify([...identity, status, assertions]);+ cases.set(key, (cases.get(key) ?? 0) + 1);+ }+ if (!cases.size) throw new Error(`${path}: no test cases`);+ if (+ integer(root!.attributes["tests"]?.value ?? String(records.length)) !==+ BigInt(records.length)+ ) {+ throw new Error(`${path}: reported test count does not match test cases`);+ }+ const failed =+ records.some(({ status }) => status === "failed") ||+ suites.some((node) =>+ ["failures", "errors"].some(+ (field) => integer(node.attributes[field]?.value ?? "0") !== 0n,+ ),+ );+ if (failed) output.error(`${path}: test run failed`);+ const duration = seconds(root!.attributes["time"]?.value ?? "0");+ const skipped = records.filter(({ status }) => status === "skipped").length;+ output.log(+ `| ${basename(path)} | ${records.length} | ${skipped} | ${duration.toFixed(2)} |`,+ );+ return {+ cases,+ duration,+ failed,+ };+}++function readReports(patterns: string[], output: Output) {+ const cases = new Map<string, number>();+ const durations: number[] = [];+ let failed = false;+ for (const pattern of patterns) {+ const paths = matchingReports(pattern).sort((a, b) =>+ Buffer.compare(Buffer.from(a), Buffer.from(b)),+ );+ if (!paths.length) throw new Error(`No reports match ${pattern}`);+ for (const path of paths) {+ const report = readReport(path, output);+ failed ||= report.failed;+ for (const [identity, count] of report.cases)+ cases.set(identity, (cases.get(identity) ?? 0) + count);+ durations.push(report.duration);+ }+ }+ return { cases, durations, failed };+}++function compareGroups(patterns: string[], output: Output): number {+ output.log("| Report | Cases | Skipped | Seconds |");+ output.log("| --- | ---: | ---: | ---: |");+ const baseline = readReports([patterns[0]!], output);+ const candidate = readReports(patterns.slice(1), output);+ let failed = baseline.failed || candidate.failed;+ for (const [label, left, right] of [+ ["Missing", baseline.cases, candidate.cases],+ ["Extra", candidate.cases, baseline.cases],+ ] as const) {+ const differences = [...left]+ .map(([identity, count]): [TestOutcome, number] => [+ JSON.parse(identity) as TestOutcome,+ count - (right.get(identity) ?? 0),+ ])+ .filter(([, count]) => count > 0)+ .sort(([a], [b]) => {+ for (let index = 0; index < a.length; index++) {+ const order = Buffer.compare(+ Buffer.from(a[index] ?? ""),+ Buffer.from(b[index] ?? ""),+ );+ if (order) return order;+ }+ return 0;+ });+ for (const [identity, count] of differences) {+ failed = true;+ const [file, classname, name, status, assertions] = identity;+ const description = [file, classname, name, status];+ if (assertions !== null) description.push(`assertions=${assertions}`);+ output.error(`${label} (${count}): ${description.join(" > ")}`);+ }+ }+ if (failed) return 1;+ if (baseline.durations.length > 1) {+ output.log(+ `\nBaseline shards: slowest ${Math.max(...baseline.durations).toFixed(2)}s; combined test time: ${baseline.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.`,+ );+ }+ output.log(+ `\nIdentical test inventory and outcomes. Slowest candidate: ${Math.max(...candidate.durations).toFixed(2)}s; combined test time: ${candidate.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.\n`,+ );+ return 0;+}++export function compareTestReports(+ argv: string[],+ output: Output = console,+): number {+ let args: { values: { help?: boolean }; positionals: string[] };+ try {+ args = parseArgs({+ args: argv,+ options: { help: { type: "boolean", short: "h" } },+ allowPositionals: true,+ });+ if (!args.values.help && args.positionals.length < 2)+ throw new Error(+ "a baseline and at least one candidate report are required",+ );+ } catch (error) {+ output.error((error as Error).message);+ return 2;+ }+ if (args.values.help) {+ output.log(+ "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nBaseline and candidates are JUnit files or glob patterns.",+ );+ return 0;+ }+ try {+ return compareGroups(args.positionals, output);+ } catch (error) {+ output.error((error as Error).message);+ return 1;+ }+}diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.tsindex e03386c24..e7a5c7ce7 100644--- a/sdk/typescript/tests-ts/test-reports.test.ts+++ b/sdk/typescript/tests-ts/test-reports.test.ts@@ -1,7 +1,8 @@import { mkdir, readFile, rm, writeFile } from "node:fs/promises";import { join } from "node:path";import { afterEach, describe, expect, test } from "bun:test";-import { bashCommand, readSubprocess } from "./support/shell.js";+import { compareTestReports } from "../scripts/test-report-comparison.mts";+import { bashCommand, runCommand } from "./support/shell.js";import { createTemporaryDirectories } from "./support/temporary-directories.js";// Node 22.13 emits this upstream notice when executing TypeScript directly.@@ -9,6 +10,12 @@ const typeStrippingNotice =/^(?:\(node:\d+\) ExperimentalWarning: Type Stripping is an experimental feature and might change at any time\n\(Use `node --trace-warnings \.\.\.` to show where the warning was created\)\n)?$/u;const bash = bashCommand();+const comparator = join(+ import.meta.dirname,+ "..",+ "scripts",+ "compare-test-reports.mts",+);const directories = createTemporaryDirectories(false);afterEach(directories.cleanup);@@ -58,18 +65,20 @@ async function workflowFixtures() {};await mkdir(join(fixture.root, "reports"));const { target } = workflow.jobs.runner.strategy.matrix;- for (const { os, shard, count, mode } of target) {- const totalCases = Math.max(- ...target.filter((item) => item.os === os).map((item) => item.count),- );- const casesPerShard = totalCases / count;- await fixture.report(- `reports/runner-${os}-${mode}-${shard}.xml`,- Array.from({ length: casesPerShard }, (_, index) =>- testcase(`${os} case ${(shard - 1) * casesPerShard + index + 1}`),- ),- );- }+ await Promise.all(+ target.map(({ os, shard, count, mode }) => {+ const totalCases = Math.max(+ ...target.filter((item) => item.os === os).map((item) => item.count),+ );+ const casesPerShard = totalCases / count;+ return fixture.report(+ `reports/runner-${os}-${mode}-${shard}.xml`,+ Array.from({ length: casesPerShard }, (_, index) =>+ testcase(`${os} case ${(shard - 1) * casesPerShard + index + 1}`),+ ),+ );+ }),+ );return {...fixture,script: workflow.jobs.compare.steps.find(@@ -84,26 +93,49 @@ function testcase(name: string, status = "", assertions?: string) {return `<testcase file="tests-ts/example.test.ts" classname="example" name="${name}"${assertionAttribute}>${status}</testcase>`;}-async function compare(baseline: string, ...candidates: string[]) {- const child = Bun.spawn({- cmd: [- "node",- "--experimental-strip-types",- join(import.meta.dirname, "..", "scripts", "compare-test-reports.mts"),- baseline,- ...candidates,- ],- stdin: "ignore",- stdout: "pipe",- stderr: "pipe",+function compare(...paths: string[]) {+ let stdout = "",+ stderr = "";+ const status = compareTestReports(paths, {+ log: (line) => {+ stdout += line + "\n";+ },+ error: (line) => {+ stderr += line + "\n";+ },});- return readSubprocess(child);+ return { status, stdout, stderr };+}++function compareCli(...paths: string[]) {+ return runCommand("node", [+ "--experimental-strip-types",+ comparator,+ ...paths,+ ]);+}++async function runWorkflow(+ fixture: Awaited<ReturnType<typeof workflowFixtures>>,+ node: string,+ env?: NodeJS.ProcessEnv,+) {+ const summary = join(fixture.root, "summary.md");+ await writeFile(summary, "");+ const result = await runCommand(+ bash,+ ["-e", "-o", "pipefail", "-c", node + "\n" + fixture.script],+ {+ cwd: fixture.root,+ env: { ...process.env, GITHUB_STEP_SUMMARY: "summary.md", ...env },+ },+ );+ return { ...result, summary: await readFile(summary, "utf8") };}describe("JUnit inventory comparison", () => {test("runs every workflow comparison before reporting a mismatch", async () => {const fixture = await workflowFixtures();- const script = fixture.script;const expected = ["ubuntu-latest", "macos-latest", "windows-latest"].map((os) =>`reports/runner-${os}-baseline-*.xml|reports/runner-${os}-expanded-*.xml`,@@ -112,27 +144,12 @@ describe("JUnit inventory comparison", () => {printf '%s\\n' "$3|$4"[[ "$3|$4" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]}`;- const summary = join(fixture.root, "summary.md");for (const failedReport of ["", expected[0]!]) {- await writeFile(summary, "");- const child = Bun.spawn({- cmd: [bash, "-e", "-o", "pipefail", "-c", `${mock}\n${script}`],- cwd: fixture.root,- stdin: "ignore",- stdout: "ignore",- stderr: "pipe",- env: {- ...process.env,- GITHUB_STEP_SUMMARY: "summary.md",- CODEX_SECURITY_TEST_FAIL_REPORT: failedReport,- },- timeout: 10_000,+ const { status, stderr, summary } = await runWorkflow(fixture, mock, {+ CODEX_SECURITY_TEST_FAIL_REPORT: failedReport,});- const { status, stderr } = await readSubprocess(child);expect(status, stderr).toBe(failedReport === "" ? 0 : 1);- expect((await readFile(summary, "utf8")).trim().split(/\r?\n/u)).toEqual(- expected,- );+ expect(summary.trim().split(/\r?\n/u)).toEqual(expected);}});@@ -147,35 +164,12 @@ describe("JUnit inventory comparison", () => {);}}- await writeFile(join(fixture.root, "summary.md"), "");- const child = Bun.spawn({- cmd: [- bash,- "-e",- "-o",- "pipefail",- "-c",- `node() { command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$3" "$4"; }\n${fixture.script}`,- ],- cwd: fixture.root,- stdin: "ignore",- stdout: "ignore",- stderr: "pipe",- env: {- ...process.env,- GITHUB_STEP_SUMMARY: "summary.md",- CODEX_SECURITY_TEST_COMPARATOR: join(- import.meta.dirname,- "..",- "scripts",- "compare-test-reports.mts",- ),- },- timeout: 10_000,- });- const { status, stderr } = await readSubprocess(child);+ const { status, stderr, summary } = await runWorkflow(+ fixture,+ 'node() { command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$3" "$4"; }',+ { CODEX_SECURITY_TEST_COMPARATOR: comparator },+ );expect(status, stderr).toBe(incomplete ? 1 : 0);- const summary = await readFile(join(fixture.root, "summary.md"), "utf8");expect(summary.match(/Identical test inventory and outcomes/gu),).toHaveLength(3);@@ -188,115 +182,61 @@ describe("JUnit inventory comparison", () => {}});- for (const scenario of [- {- name: "missing baseline Linux shard",- os: "ubuntu-latest",- mode: "baseline",- count: 3,- missing: 3,- },- {- name: "missing expanded Linux shard",- os: "ubuntu-latest",- mode: "expanded",- count: 6,- missing: 6,- },- {- name: "missing baseline Windows shard",- os: "windows-latest",- mode: "baseline",- count: 7,- missing: 7,- },- {- name: "missing expanded Windows shard",- os: "windows-latest",- mode: "expanded",- count: 14,- missing: 14,- },- {- name: "missing baseline shard replaced by an unexpected index",- os: "ubuntu-latest",- mode: "baseline",- count: 3,- missing: 3,- extra: 4,- },- {- name: "missing expanded shard replaced by an unexpected index",- os: "ubuntu-latest",- mode: "expanded",- count: 6,- missing: 6,- extra: 7,- },- {- name: "extra baseline shard",- os: "windows-latest",- mode: "baseline",- count: 7,- extra: 8,- },- {- name: "extra expanded shard",- os: "windows-latest",- mode: "expanded",- count: 14,- extra: 15,- },- ]) {- test(`rejects incomplete workflow groups: ${scenario.name}`, async () => {+ for (const [name, os, mode, count, issue] of [+ ["missing baseline Linux shard", "ubuntu-latest", "baseline", 3, "missing"],+ ["missing expanded Linux shard", "ubuntu-latest", "expanded", 6, "missing"],+ [+ "missing baseline Windows shard",+ "windows-latest",+ "baseline",+ 7,+ "missing",+ ],+ [+ "missing expanded Windows shard",+ "windows-latest",+ "expanded",+ 14,+ "missing",+ ],+ [+ "missing baseline shard replaced by an unexpected index",+ "ubuntu-latest",+ "baseline",+ 3,+ "replaced",+ ],+ [+ "missing expanded shard replaced by an unexpected index",+ "ubuntu-latest",+ "expanded",+ 6,+ "replaced",+ ],+ ["extra baseline shard", "windows-latest", "baseline", 7, "extra"],+ ["extra expanded shard", "windows-latest", "expanded", 14, "extra"],+ ] as const) {+ test(`rejects incomplete workflow groups: ${name}`, async () => {const fixture = await workflowFixtures();- if (scenario.missing !== undefined) {- await rm(- join(- fixture.root,- `reports/runner-${scenario.os}-${scenario.mode}-${scenario.missing}.xml`,- ),- );- }- if (scenario.extra !== undefined) {- await fixture.report(- `reports/runner-${scenario.os}-${scenario.mode}-${scenario.extra}.xml`,- [testcase("unexpected shard")],- );- }- const child = Bun.spawn({- cmd: [- bash,- "-e",- "-o",- "pipefail",- "-c",- `node() { printf '%s\\n' "$3|$4"; }\n${fixture.script}`,- ],- cwd: fixture.root,- stdin: "ignore",- stdout: "ignore",- stderr: "pipe",- env: { ...process.env, GITHUB_STEP_SUMMARY: "summary.md" },- timeout: 10_000,- });- const { status, stderr } = await readSubprocess(child);+ const prefix = `reports/runner-${os}-${mode}-`;+ if (issue !== "extra")+ await rm(join(fixture.root, `${prefix}${count}.xml`));+ if (issue !== "missing")+ await fixture.report(`${prefix}${count + 1}.xml`, [+ testcase("unexpected shard"),+ ]);+ const { status, stderr, summary } = await runWorkflow(+ fixture,+ `node() { printf '%s\\n' "$3|$4"; }`,+ );expect(status, stderr).toBe(1);- if (scenario.missing !== undefined) {- expect(stderr).toContain(- `Missing shard report: reports/runner-${scenario.os}-${scenario.mode}-${scenario.missing}.xml`,- );- } else {- expect(stderr).toContain(- `Expected ${scenario.count} reports for ${scenario.os}/${scenario.mode}, found ${scenario.count + 1}`,- );- }+ expect(stderr).toContain(+ issue !== "extra"+ ? `Missing shard report: ${prefix}${count}.xml`+ : `Expected ${count} reports for ${os}/${mode}, found ${count + 1}`,+ );// Missing artifacts must not prevent the other OS/mode diagnostics.- expect(- (await readFile(join(fixture.root, "summary.md"), "utf8"))- .trim()- .split(/\r?\n/u),- ).toHaveLength(3);+ expect(summary.trim().split(/\r?\n/u)).toHaveLength(3);});}@@ -307,7 +247,10 @@ describe("JUnit inventory comparison", () => {const baseline = await fixture.report("baseline.xml", [passed, skipped]);await fixture.report("shard-1.xml", [skipped]);await fixture.report("shard-2.xml", [passed]);- const result = await compare(baseline, join(fixture.root, "shard-*.xml"));+ const result = await compareCli(+ baseline,+ join(fixture.root, "shard-*.xml"),+ );expect(result.status, result.stderr).toBe(0);expect(result.stderr).toMatch(typeStrippingNotice);expect(result.stdout).toBe(@@ -335,7 +278,7 @@ describe("JUnit inventory comparison", () => {${testcase("a & " ' < >")}</testsuite></testsuites>`,);- const result = await compare(baseline, candidate);+ const result = compare(baseline, candidate);expect(result.status, result.stderr).toBe(0);expect(result.stderr).toMatch(typeStrippingNotice);});@@ -350,8 +293,8 @@ ${testcase("a & " ' < >")}"./tests-ts\\example.test.ts",),]);- expect((await compare(baseline, candidate)).status).toBe(0);- const result = await compare(baseline, candidate, candidate);+ expect(compare(baseline, candidate).status).toBe(0);+ const result = await compareCli(baseline, candidate, candidate);expect(result.status).toBe(1);expect(result.stderr).toContain("Extra (1): tests-ts/example.test.ts > example > portable > passed",@@ -379,16 +322,16 @@ ${testcase("a & " ' < >")}"reports with spaces/*.xml","**/shard-*.xml",]) {- const result = await compare(baseline, join(fixture.root, pattern));+ const result = compare(baseline, join(fixture.root, pattern));expect(result.status, `${pattern}: ${result.stderr}`).toBe(0);}const hidden = await fixture.report("hidden-baseline.xml", [testcase("hidden"),]);- expect((await compare(hidden, join(directory, ".*.xml"))).status).toBe(0);+ expect(compare(hidden, join(directory, ".*.xml")).status).toBe(0);for (const name of ["{a,b}.xml", "!report.xml", "@(shard).xml"]) {const path = await fixture.report(name, cases);- expect((await compare(baseline, path)).status, name).toBe(0);+ expect(compare(baseline, path).status, name).toBe(0);}});@@ -401,7 +344,7 @@ ${testcase("a & " ' < >")}const candidate = await fixture.report("error.xml", [testcase("example", status),]);- const result = await compare(baseline, candidate);+ const result = compare(baseline, candidate);expect(result.status).toBe(1);expect(result.stderr).toContain("test run failed");expect(result.stderr).toContain("example > example > failed");@@ -412,7 +355,7 @@ ${testcase("a & " ' < >")}nested,`<testsuites tests="1"><testsuite><testsuite errors="1">${testcase("example")}</testsuite></testsuite></testsuites>`,);- const result = await compare(baseline, nested);+ const result = compare(baseline, nested);expect(result.status).toBe(1);expect(result.stderr).toContain("test run failed");});@@ -434,7 +377,10 @@ ${testcase("a & " ' < >")}]) {const candidate = join(fixture.root, "invalid.xml");await writeFile(candidate, xml);- const result = await compare(baseline, candidate);+ const result =+ xml === "<testsuite><testcase/></testsuite><extra/>"+ ? await compareCli(baseline, candidate)+ : compare(baseline, candidate);expect(result.status, xml).toBe(1);expect(result.stdout).not.toContain("Identical test inventory");}@@ -450,7 +396,7 @@ ${testcase("a & " ' < >")}candidate,`<testsuite xmlns:other="urn:example">${testcase("example", "<properties><failure/></properties><other:error/>")}</testsuite>`,);- const result = await compare(baseline, candidate);+ const result = compare(baseline, candidate);expect(result.status, result.stderr).toBe(0);expect(result.stdout).toContain("| defaults.xml | 1 | 0 | 0.00 |");});@@ -469,7 +415,7 @@ ${testcase("a & " ' < >")}passed,passed,]);- const result = await compare(baseline, candidate);+ const result = compare(baseline, candidate);expect(result.status, result.stderr).toBe(0);expect(result.stdout).toContain("| baseline.xml | 3 | 1 | 1.25 |");for (const [name, cases, diagnostic] of [@@ -478,7 +424,7 @@ ${testcase("a & " ' < >")}["changed", [passed, skipped, skipped], "Missing (1)"],] as const) {const changed = await fixture.report(`${name}.xml`, [...cases]);- const comparison = await compare(baseline, changed);+ const comparison = compare(baseline, changed);expect(comparison.status, name).toBe(1);expect(comparison.stderr).toContain(diagnostic);}@@ -494,7 +440,7 @@ ${testcase("a & " ' < >")}await fixture.report("candidate-2.xml", [first]);const baseline = join(fixture.root, "baseline-*.xml");const candidate = join(fixture.root, "candidate-*.xml");- const result = await compare(baseline, candidate);+ const result = compare(baseline, candidate);expect(result.status, result.stderr).toBe(0);expect(result.stdout).toContain("Baseline shards: slowest 1.25s; combined test time: 2.50s.",@@ -503,18 +449,15 @@ ${testcase("a & " ' < >")}"Slowest candidate: 1.25s; combined test time: 2.50s.",);const extra = await fixture.report("extra.xml", [repeated]);- const duplicated = await compare(baseline, candidate, extra);+ const duplicated = compare(baseline, candidate, extra);expect(duplicated.status).toBe(1);expect(duplicated.stderr).toContain("Extra (1)");- const missing = await compare(- baseline,- join(fixture.root, "candidate-1.xml"),- );+ const missing = compare(baseline, join(fixture.root, "candidate-1.xml"));expect(missing.status).toBe(1);expect(missing.stderr).toContain("Missing (1)");- expect(- (await compare(join(fixture.root, "absent-*.xml"), candidate)).status,- ).toBe(1);+ expect(compare(join(fixture.root, "absent-*.xml"), candidate).status).toBe(+ 1,+ );});test("preserves assertion counts for repeated identities across different shard layouts", async () => {@@ -528,11 +471,11 @@ ${testcase("a & " ' < >")}await fixture.report("candidate-3.xml", [repeated("05")]);const baseline = join(fixture.root, "baseline-*.xml");const candidate = join(fixture.root, "candidate-*.xml");- const matching = await compare(baseline, candidate);+ const matching = compare(baseline, candidate);expect(matching.status, matching.stderr).toBe(0);await fixture.report("candidate-1.xml", [repeated("4")]);- const changed = await compare(baseline, candidate);+ const changed = compare(baseline, candidate);expect(changed.status).toBe(1);expect(changed.stderr).toContain("Missing (1): tests-ts/example.test.ts > example > same parameterized name > passed > assertions=5",@@ -550,7 +493,7 @@ ${testcase("a & " ' < >")}const candidate = await fixture.report("candidate.xml", [testcase("without assertions"),]);- const result = await compare(baseline, candidate);+ const result = compare(baseline, candidate);expect(result.status).toBe(1);expect(result.stderr).toContain("Missing (1): tests-ts/example.test.ts > example > without assertions > passed > assertions=0",@@ -569,7 +512,7 @@ ${testcase("a & " ' < >")}testcase("first", "<failure/>"),]);const passed = await fixture.report("shard-2.xml", [second]);- const result = await compare(baseline, failed, passed);+ const result = compare(baseline, failed, passed);expect(result.status).toBe(1);expect(result.stderr).toContain("test run failed");@@ -597,10 +540,10 @@ ${testcase("a & " ' < >")}failures,count,);- expect((await compare(baseline, candidate)).status, name).toBe(1);+ expect(compare(baseline, candidate).status, name).toBe(1);}- expect(- (await compare(baseline, join(fixture.root, "absent-*.xml"))).status,- ).toBe(1);+ expect(compare(baseline, join(fixture.root, "absent-*.xml")).status).toBe(+ 1,+ );});});