OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
test(ci): pass JUnit shard files directly to the comparator (#2250)
Michael D'Angeloc40e7bbf3857
diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.ymlindex a6bd1ba25..de21b4711 100644--- a/.github/workflows/test-quality.yml+++ b/.github/workflows/test-quality.yml@@ -201,8 +201,13 @@ jobs:for mode in baseline expanded; doshards=3if [[ "$os" == windows-latest ]]; then shards=7; fi- if [[ "$mode" == expanded ]]; then shards=$((shards * 2)); fireports=(reports/runner-"$os"-"$mode"-*.xml)+ if [[ "$mode" == expanded ]]; then+ shards=$((shards * 2))+ expanded_reports=("${reports[@]}")+ else+ baseline_reports=("${reports[@]}")+ fiif (( ${#reports[@]} != shards )); thenecho "Expected $shards reports for $os/$mode, found ${#reports[@]}" >&2comparison_status=1@@ -215,7 +220,7 @@ jobs:fidonedone- node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "reports/runner-$os-baseline-*.xml" "reports/runner-$os-expanded-*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1+ node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "${baseline_reports[@]}" -- "${expanded_reports[@]}" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1doneexit "$comparison_status"diff --git a/sdk/typescript/package.json b/sdk/typescript/package.jsonindex bcebeadd3..a97b00433 100644--- a/sdk/typescript/package.json+++ b/sdk/typescript/package.json@@ -116,7 +116,6 @@"ink-testing-library": "4.0.0","json-schema-to-typescript": "16.0.0","json5": "2.2.3",- "minimatch": "10.2.6","prettier": "3.9.10","typescript": "7.0.2"}diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yamlindex 0b73b5310..954fb1d2a 100644--- a/sdk/typescript/pnpm-lock.yaml+++ b/sdk/typescript/pnpm-lock.yaml@@ -113,9 +113,6 @@ importers:json5:specifier: 2.2.3version: 2.2.3- minimatch:- specifier: 10.2.6- version: 10.2.6prettier:specifier: 3.9.10version: 3.9.10diff --git a/sdk/typescript/scripts/test-report-comparison.mts b/sdk/typescript/scripts/test-report-comparison.mtsindex 3dbf45769..175bbb7ac 100644--- a/sdk/typescript/scripts/test-report-comparison.mts+++ b/sdk/typescript/scripts/test-report-comparison.mts@@ -1,8 +1,7 @@// Compare Bun JUnit inventories before changing the required CI runner.-import { lstatSync, readFileSync, readdirSync, statSync } from "node:fs";-import { basename, dirname, join, sep } from "node:path";+import { readFileSync } from "node:fs";+import { basename } from "node:path";import { parseArgs } from "node:util";-import { minimatch } from "minimatch";import { SaxesParser, type SaxesTagNS } from "saxes";type Output = Pick<Console, "log" | "error">;@@ -17,59 +16,6 @@ type TestRecord = {attributes: SaxesTagNS["attributes"];status: TestStatus;};-function matchingReports(pattern: string): string[] {- // Python glob treats ** as one component and does not expand braces/extglobs.- const directoriesOnly =- pattern.endsWith(sep) ||- (process.platform === "win32" && pattern.endsWith("/"));- if (!/[*?[]/u.test(pattern)) {- try {- const stat = directoriesOnly ? statSync(pattern) : lstatSync(pattern);- return !directoriesOnly || stat.isDirectory() ? [pattern] : [];- } catch {- return [];- }- }- const parent = dirname(pattern);- const namePattern = basename(pattern);- return (/[*?[]/u.test(parent) ? matchingReports(parent) : [parent]).flatMap(- (directory) => {- let names: string[];- try {- names = readdirSync(directory);- } catch {- return [];- }- return names- .filter(- (name) =>- (!name.startsWith(".") || namePattern.startsWith(".")) &&- minimatch(- name,- namePattern.replaceAll("\\", "\\\\").replaceAll("[^", "[\\^"),- {- dot: true,- nobrace: true,- noext: true,- noglobstar: true,- nonegate: true,- nocomment: true,- nocase: process.platform === "win32",- },- ),- )- .map((name) => join(directory, name))- .filter((path) => {- if (!directoriesOnly) return true;- try {- return statSync(path).isDirectory();- } catch {- return false;- }- });- },- );-}function integer(value: string): bigint {if (!/^[+-]?\d(?:_?\d)*$/u.test(value.trim())) {@@ -163,31 +109,30 @@ function readReport(path: string, output: Output) {};}-function readReports(patterns: string[], output: Output) {+function readReports(paths: string[], group: string, output: Output) {+ if (!paths.length) throw new Error(`No ${group} reports supplied`);const cases = new Map<string, number>();const durations: number[] = [];let failed = false;- for (const pattern of patterns) {- const paths = matchingReports(pattern).sort((a, b) =>- Buffer.compare(Buffer.from(a), Buffer.from(b)),- );- if (!paths.length) throw new Error(`No reports match ${pattern}`);- for (const path of paths) {- const report = readReport(path, output);- failed ||= report.failed;- for (const [identity, count] of report.cases)- cases.set(identity, (cases.get(identity) ?? 0) + count);- durations.push(report.duration);- }+ for (const path of paths) {+ const report = readReport(path, output);+ failed ||= report.failed;+ for (const [identity, count] of report.cases)+ cases.set(identity, (cases.get(identity) ?? 0) + count);+ durations.push(report.duration);}return { cases, durations, failed };}-function compareGroups(patterns: string[], output: Output): number {+function compareGroups(+ baselinePaths: string[],+ candidatePaths: string[],+ output: Output,+): number {output.log("| Report | Cases | Skipped | Seconds |");output.log("| --- | ---: | ---: | ---: |");- const baseline = readReports([patterns[0]!], output);- const candidate = readReports(patterns.slice(1), output);+ const baseline = readReports(baselinePaths, "baseline", output);+ const candidate = readReports(candidatePaths, "candidate", output);let failed = baseline.failed || candidate.failed;for (const [label, left, right] of [["Missing", baseline.cases, candidate.cases],@@ -233,29 +178,34 @@ export function compareTestReports(argv: string[],output: Output = console,): number {+ const separator = argv.indexOf("--");let args: { values: { help?: boolean }; positionals: string[] };try {args = parseArgs({- args: argv,+ args: separator === -1 ? argv : argv.slice(0, separator),options: { help: { type: "boolean", short: "h" } },allowPositionals: true,});- if (!args.values.help && args.positionals.length < 2)- throw new Error(- "a baseline and at least one candidate report are required",- );} catch (error) {output.error((error as Error).message);return 2;}if (args.values.help) {output.log(- "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nBaseline and candidates are JUnit files or glob patterns.",+ "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidate [candidate ...]\n or: node --experimental-strip-types compare-test-reports.mts baseline [baseline ...] -- candidate [candidate ...]\nPass JUnit filenames. Your shell can expand wildcard patterns.",);return 0;}+ if (separator === -1 && args.positionals.length < 2) {+ output.error("a baseline and at least one candidate report are required");+ return 2;+ }+ const baselinePaths =+ separator === -1 ? args.positionals.slice(0, 1) : args.positionals;+ const candidatePaths =+ separator === -1 ? args.positionals.slice(1) : argv.slice(separator + 1);try {- return compareGroups(args.positionals, output);+ return compareGroups(baselinePaths, candidatePaths, output);} catch (error) {output.error((error as Error).message);return 1;diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.tsindex e7a5c7ce7..58955b8e4 100644--- a/sdk/typescript/tests-ts/test-reports.test.ts+++ b/sdk/typescript/tests-ts/test-reports.test.ts@@ -137,12 +137,11 @@ describe("JUnit inventory comparison", () => {test("runs every workflow comparison before reporting a mismatch", async () => {const fixture = await workflowFixtures();const expected = ["ubuntu-latest", "macos-latest", "windows-latest"].map(- (os) =>- `reports/runner-${os}-baseline-*.xml|reports/runner-${os}-expanded-*.xml`,+ (os) => `reports/runner-${os}-baseline-1.xml`,);const mock = `node() {- printf '%s\\n' "$3|$4"- [[ "$3|$4" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]+ printf '%s\\n' "$3"+ [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]}`;for (const failedReport of ["", expected[0]!]) {const { status, stderr, summary } = await runWorkflow(fixture, mock, {@@ -166,7 +165,7 @@ describe("JUnit inventory comparison", () => {}const { status, stderr, summary } = await runWorkflow(fixture,- 'node() { command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$3" "$4"; }',+ 'node() { shift 2; command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$@"; }',{ CODEX_SECURITY_TEST_COMPARATOR: comparator },);expect(status, stderr).toBe(incomplete ? 1 : 0);@@ -227,7 +226,7 @@ describe("JUnit inventory comparison", () => {]);const { status, stderr, summary } = await runWorkflow(fixture,- `node() { printf '%s\\n' "$3|$4"; }`,+ "node() { echo compared; }",);expect(status, stderr).toBe(1);expect(stderr).toContain(@@ -245,12 +244,9 @@ describe("JUnit inventory comparison", () => {const passed = testcase("accepts & preserves");const skipped = testcase("platform-only", "<skipped/>");const baseline = await fixture.report("baseline.xml", [passed, skipped]);- await fixture.report("shard-1.xml", [skipped]);- await fixture.report("shard-2.xml", [passed]);- const result = await compareCli(- baseline,- join(fixture.root, "shard-*.xml"),- );+ const first = await fixture.report("shard-1.xml", [skipped]);+ const second = await fixture.report("shard-2.xml", [passed]);+ const result = await compareCli(baseline, first, second);expect(result.status, result.stderr).toBe(0);expect(result.stderr).toMatch(typeStrippingNotice);expect(result.stdout).toBe(@@ -301,40 +297,40 @@ ${testcase("a & " ' < >")});});- test("preserves file, character-class, hidden-file, and nonrecursive glob matching", async () => {+ test("reads literal report paths containing spaces and brackets", async () => {const fixture = await fixtures();- const cases = [testcase("portable")];+ const cases = [testcase("selected")];const baseline = await fixture.report("baseline.xml", cases);- const directory = join(fixture.root, "reports with spaces");- await mkdir(directory);- await fixture.report("reports with spaces/shard-a.xml", cases);- await fixture.report("reports with spaces/.hidden.xml", [- testcase("hidden"),- ]);- await mkdir(join(directory, "nested"));- await fixture.report("reports with spaces/nested/shard-b.xml", [- testcase("nested"),- ]);- for (const pattern of [- "reports with spaces/shard-?.xml",- "reports with spaces/shard-[ab].xml",- "reports with spaces/shard-[!b].xml",- "reports with spaces/*.xml",- "**/shard-*.xml",+ const literal = await fixture.report("shard [ab].xml", cases);+ await fixture.report("shard a.xml", [testcase("unrelated")]);+ const result = compare(baseline, literal);+ expect(result.status, result.stderr).toBe(0);+ expect(result.stdout).toContain("| shard [ab].xml |");++ const absent = join(fixture.root, "shard *.xml");+ for (const args of [+ [baseline, absent],+ [absent, baseline],]) {- const result = compare(baseline, join(fixture.root, pattern));- expect(result.status, `${pattern}: ${result.stderr}`).toBe(0);- }- const hidden = await fixture.report("hidden-baseline.xml", [- testcase("hidden"),- ]);- expect(compare(hidden, join(directory, ".*.xml")).status).toBe(0);- for (const name of ["{a,b}.xml", "!report.xml", "@(shard).xml"]) {- const path = await fixture.report(name, cases);- expect(compare(baseline, path).status, name).toBe(0);+ const missing = compare(...args);+ expect(missing.status).toBe(1);+ expect(missing.stderr).toContain(absent);}});+ test("reports available baseline timings when the candidate group is empty", async () => {+ const fixture = await fixtures();+ const report = await fixture.report("shard.xml", [testcase("present")]);+ const noBaseline = compare("--", report);+ expect(noBaseline.status).toBe(1);+ expect(noBaseline.stderr).toContain("No baseline reports supplied");++ const noCandidate = compare(report, "--");+ expect(noCandidate.status).toBe(1);+ expect(noCandidate.stderr).toContain("No candidate reports supplied");+ expect(noCandidate.stdout).toContain("| shard.xml | 1 | 0 | 1.25 |");+ });+test("detects error status and nested summary failures with failure overriding skipped", async () => {const fixture = await fixtures();const baseline = await fixture.report("baseline.xml", [@@ -434,13 +430,16 @@ ${testcase("a & " ' < >")}const fixture = await fixtures();const first = testcase("first");const repeated = testcase("parameterized name");- await fixture.report("baseline-1.xml", [first, repeated]);- await fixture.report("baseline-2.xml", [repeated]);- await fixture.report("candidate-1.xml", [repeated, repeated]);- await fixture.report("candidate-2.xml", [first]);- const baseline = join(fixture.root, "baseline-*.xml");- const candidate = join(fixture.root, "candidate-*.xml");- const result = compare(baseline, candidate);+ const baseline = [+ await fixture.report("baseline-1.xml", [first, repeated]),+ await fixture.report("baseline-2.xml", [repeated]),+ ];+ const firstCandidate = await fixture.report("candidate-1.xml", [+ repeated,+ repeated,+ ]);+ const secondCandidate = await fixture.report("candidate-2.xml", [first]);+ const result = compare(...baseline, "--", firstCandidate, secondCandidate);expect(result.status, result.stderr).toBe(0);expect(result.stdout).toContain("Baseline shards: slowest 1.25s; combined test time: 2.50s.",@@ -449,33 +448,38 @@ ${testcase("a & " ' < >")}"Slowest candidate: 1.25s; combined test time: 2.50s.",);const extra = await fixture.report("extra.xml", [repeated]);- const duplicated = compare(baseline, candidate, extra);+ const duplicated = compare(+ ...baseline,+ "--",+ firstCandidate,+ secondCandidate,+ extra,+ );expect(duplicated.status).toBe(1);expect(duplicated.stderr).toContain("Extra (1)");- const missing = compare(baseline, join(fixture.root, "candidate-1.xml"));+ const missing = compare(...baseline, "--", firstCandidate);expect(missing.status).toBe(1);expect(missing.stderr).toContain("Missing (1)");- expect(compare(join(fixture.root, "absent-*.xml"), candidate).status).toBe(- 1,- );});test("preserves assertion counts for repeated identities across different shard layouts", async () => {const fixture = await fixtures();const repeated = (assertions: string) =>testcase("same parameterized name", "", assertions);- await fixture.report("baseline-1.xml", [repeated("2"), repeated("5")]);- await fixture.report("baseline-2.xml", [repeated("5")]);- await fixture.report("candidate-1.xml", [repeated("5")]);- await fixture.report("candidate-2.xml", [repeated("2")]);- await fixture.report("candidate-3.xml", [repeated("05")]);- const baseline = join(fixture.root, "baseline-*.xml");- const candidate = join(fixture.root, "candidate-*.xml");- const matching = compare(baseline, candidate);+ const baseline = [+ await fixture.report("baseline-1.xml", [repeated("2"), repeated("5")]),+ await fixture.report("baseline-2.xml", [repeated("5")]),+ ];+ const candidate = [+ await fixture.report("candidate-1.xml", [repeated("5")]),+ await fixture.report("candidate-2.xml", [repeated("2")]),+ await fixture.report("candidate-3.xml", [repeated("05")]),+ ];+ const matching = compare(...baseline, "--", ...candidate);expect(matching.status, matching.stderr).toBe(0);await fixture.report("candidate-1.xml", [repeated("4")]);- const changed = compare(baseline, candidate);+ const changed = compare(...baseline, "--", ...candidate);expect(changed.status).toBe(1);expect(changed.stderr).toContain("Missing (1): tests-ts/example.test.ts > example > same parameterized name > passed > assertions=5",@@ -542,8 +546,5 @@ ${testcase("a & " ' < >")});expect(compare(baseline, candidate).status, name).toBe(1);}- expect(compare(baseline, join(fixture.root, "absent-*.xml")).status).toBe(- 1,- );});});