Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

ci: expand balanced test shards on the stable runner (#1897)

Michael D'Angelo28b238618522
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index c37f0da58..0a8c71c72 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -201,7 +201,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
- shard: [1, 2, 3]
+ shard: [1, 2, 3, 4, 5, 6]
steps:
- *checkout-repository
- *download-native
@@ -232,7 +232,7 @@ jobs:
TMP: ${{ runner.temp }}
TMPDIR: ${{ runner.temp }}
CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
- run: node --experimental-strip-types scripts/run-ci-tests.mts ${{ matrix.shard }}/3 ${{ matrix.os == 'ubuntu-latest' && '--coverage --coverage-reporter=text --coverage-reporter=lcov' || '' }}
+ run: node --experimental-strip-types scripts/run-ci-tests.mts ${{ matrix.shard }}/6 ${{ matrix.os == 'ubuntu-latest' && '--coverage --coverage-reporter=text --coverage-reporter=lcov' || '' }}
- name: Upload test reports
if: always()
continue-on-error: true
@@ -515,7 +515,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- shard: [1, 2, 3, 4, 5, 6, 7]
+ shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
steps:
- *checkout-repository
- &set-up-python
@@ -548,7 +548,7 @@ jobs:
TMP: ${{ steps.windows-temp.outputs.path }}
TMPDIR: ${{ steps.windows-temp.outputs.path }}
CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
- run: node --experimental-strip-types sdk/typescript/scripts/run-ci-tests.mts ${{ matrix.shard }}/7 --seed=${{ github.run_number }}
+ run: node --experimental-strip-types sdk/typescript/scripts/run-ci-tests.mts ${{ matrix.shard }}/14 --seed=${{ github.run_number }}
- name: Upload Windows test reports
if: always()
continue-on-error: true
diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml
index 011949f14..26c275e1c 100644
--- a/.github/workflows/test-quality.yml
+++ b/.github/workflows/test-quality.yml
@@ -30,7 +30,7 @@ jobs:
runner:
needs: native
if: ${{ !cancelled() && (inputs.native-artifacts-ready || needs.native.result == 'success') }}
- name: ${{ matrix.target.os }} / ${{ matrix.mode.name }} / shard-${{ matrix.target.shard }}
+ name: ${{ matrix.target.os }} / ${{ matrix.target.mode }} / shard-${{ matrix.target.shard }}
runs-on: ${{ matrix.target.os }}
timeout-minutes: ${{ matrix.target.os == 'windows-latest' && 90 || 30 }}
env:
@@ -38,25 +38,47 @@ jobs:
strategy:
fail-fast: false
matrix:
- # Every mode uses the same production shard count and timing snapshot.
+ # Compare the previous layout with the expanded production layout.
target:
- - { os: ubuntu-latest, shard: 1, count: 3 }
- - { os: ubuntu-latest, shard: 2, count: 3 }
- - { os: ubuntu-latest, shard: 3, count: 3 }
- - { os: macos-latest, shard: 1, count: 3 }
- - { os: macos-latest, shard: 2, count: 3 }
- - { os: macos-latest, shard: 3, count: 3 }
- - { os: windows-latest, shard: 1, count: 7 }
- - { os: windows-latest, shard: 2, count: 7 }
- - { os: windows-latest, shard: 3, count: 7 }
- - { os: windows-latest, shard: 4, count: 7 }
- - { os: windows-latest, shard: 5, count: 7 }
- - { os: windows-latest, shard: 6, count: 7 }
- - { os: windows-latest, shard: 7, count: 7 }
- mode:
- - { name: baseline, args: "" }
- - { name: isolated, args: --isolate }
- - { name: parallel, args: --parallel=2 }
+ - { os: ubuntu-latest, shard: 1, count: 3, mode: baseline }
+ - { os: ubuntu-latest, shard: 2, count: 3, mode: baseline }
+ - { os: ubuntu-latest, shard: 3, count: 3, mode: baseline }
+ - { os: macos-latest, shard: 1, count: 3, mode: baseline }
+ - { os: macos-latest, shard: 2, count: 3, mode: baseline }
+ - { os: macos-latest, shard: 3, count: 3, mode: baseline }
+ - { os: windows-latest, shard: 1, count: 7, mode: baseline }
+ - { os: windows-latest, shard: 2, count: 7, mode: baseline }
+ - { os: windows-latest, shard: 3, count: 7, mode: baseline }
+ - { os: windows-latest, shard: 4, count: 7, mode: baseline }
+ - { os: windows-latest, shard: 5, count: 7, mode: baseline }
+ - { os: windows-latest, shard: 6, count: 7, mode: baseline }
+ - { os: windows-latest, shard: 7, count: 7, mode: baseline }
+ - { os: ubuntu-latest, shard: 1, count: 6, mode: expanded }
+ - { os: ubuntu-latest, shard: 2, count: 6, mode: expanded }
+ - { os: ubuntu-latest, shard: 3, count: 6, mode: expanded }
+ - { os: ubuntu-latest, shard: 4, count: 6, mode: expanded }
+ - { os: ubuntu-latest, shard: 5, count: 6, mode: expanded }
+ - { os: ubuntu-latest, shard: 6, count: 6, mode: expanded }
+ - { os: macos-latest, shard: 1, count: 6, mode: expanded }
+ - { os: macos-latest, shard: 2, count: 6, mode: expanded }
+ - { os: macos-latest, shard: 3, count: 6, mode: expanded }
+ - { os: macos-latest, shard: 4, count: 6, mode: expanded }
+ - { os: macos-latest, shard: 5, count: 6, mode: expanded }
+ - { os: macos-latest, shard: 6, count: 6, mode: expanded }
+ - { os: windows-latest, shard: 1, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 2, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 3, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 4, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 5, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 6, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 7, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 8, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 9, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 10, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 11, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 12, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 13, count: 14, mode: expanded }
+ - { os: windows-latest, shard: 14, count: 14, mode: expanded }
steps:
- &actions-checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -82,8 +104,7 @@ jobs:
plugins/codex-security/mcp-app/pnpm-lock.yaml
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
- # Includes the isolated spawnSync fix from oven-sh/bun#44581.
- bun-version: "1.4.3"
+ bun-version: "1.3.14"
- name: Install dependencies
run: |
pnpm --dir sdk/typescript install --frozen-lockfile
@@ -100,20 +121,20 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install --yes ripgrep
- - name: Test runner mode
+ - name: Test shard layout
working-directory: sdk/typescript
env:
TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
TMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }}
- run: node --experimental-strip-types scripts/run-ci-tests.mts ${{ matrix.target.shard }}/${{ matrix.target.count }} ${{ matrix.mode.args }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter-outfile=reports/runner-${{ matrix.target.os }}-${{ matrix.mode.name }}-${{ matrix.target.shard }}.xml
+ run: node --experimental-strip-types scripts/run-ci-tests.mts ${{ matrix.target.shard }}/${{ matrix.target.count }} ${{ matrix.target.os == 'ubuntu-latest' && '--coverage --coverage-reporter=lcov' || '' }} --coverage-dir=reports/coverage-${{ matrix.target.os }}-${{ matrix.target.mode }}-${{ matrix.target.shard }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter-outfile=reports/runner-${{ matrix.target.os }}-${{ matrix.target.mode }}-${{ matrix.target.shard }}.xml
- name: Upload runner report
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
- name: runner-${{ matrix.target.os }}-${{ matrix.mode.name }}-${{ matrix.target.shard }}
+ name: runner-${{ matrix.target.os }}-${{ matrix.target.mode }}-${{ matrix.target.shard }}
overwrite: true
- path: sdk/typescript/reports/runner-*.xml
+ path: sdk/typescript/reports/
if-no-files-found: error
retention-days: 14
@@ -142,9 +163,10 @@ jobs:
comparison_status=0
shopt -s nullglob
for os in ubuntu-latest macos-latest windows-latest; do
- shards=3
- if [[ "$os" == windows-latest ]]; then shards=7; fi
- for mode in baseline isolated parallel; do
+ for mode in baseline expanded; do
+ shards=3
+ if [[ "$os" == windows-latest ]]; then shards=7; fi
+ if [[ "$mode" == expanded ]]; then shards=$((shards * 2)); fi
reports=(reports/runner-"$os"-"$mode"-*.xml)
if (( ${#reports[@]} != shards )); then
echo "Expected $shards reports for $os/$mode, found ${#reports[@]}" >&2
@@ -158,9 +180,7 @@ jobs:
fi
done
done
- for mode in isolated parallel; do
- node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "reports/runner-$os-baseline-*.xml" "reports/runner-$os-$mode-*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
- done
+ node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "reports/runner-$os-baseline-*.xml" "reports/runner-$os-expanded-*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
done
exit "$comparison_status"
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index fe5c8a165..b027151e3 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -44,7 +44,7 @@ Tests run in random order by default. To replay a failure, pass the seed from
Bun's summary to `pnpm run test --seed 12345`.
The local test commands pass a 30-second per-test timeout explicitly. Windows
-CI and the Windows runner experiment allow 120 seconds for slower native
+CI and the Windows runner diagnostics allow 120 seconds for slower native
credential and document checks. `test:ci` writes `reports/junit.xml` and
`coverage/lcov.info`. Coverage measures loaded
JavaScript and TypeScript, not the Python helpers or child processes. It is
@@ -142,11 +142,12 @@ Typechecking and formatting run once in an independent required job, so package
consumers do not wait for those checks. Package compilation and archive
validation still finish before the test jobs start.
-The full Bun suite runs once per OS under Node 22: three file shards on Linux
-and macOS, and seven on Windows. The other Node versions run the installed
-package checks instead of repeating the same Bun suite. MCP and Python tests
-run in separate required jobs. Python uses four isolated pytest-xdist workers
-with work stealing; worker crashes fail the run without automatic restarts.
+The full Bun suite runs once per OS under Node 22: six file shards on Linux
+and macOS, and fourteen on Windows. GitHub runs the shards as separate jobs;
+each job uses Bun's default serial runner. The other Node versions run the
+installed-package checks instead of repeating the same Bun suite. MCP and
+Python tests run in separate required jobs. Python uses four isolated
+pytest-xdist workers with work stealing; worker crashes fail the run without automatic restarts.
`scripts/run-ci-tests.mts` discovers every shared test file and balances whole
files using the platform's recorded durations in `scripts/test-timings.json`.
@@ -156,7 +157,7 @@ snapshot. New files are included automatically with the mean known-file duration
deleted files in the timing snapshot are ignored. Timing hints never select or
exclude tests, and adding or splitting tests does not require updating them.
To reproduce one Windows shard locally after building the plugin, run
-`node --experimental-strip-types scripts/run-ci-tests.mts 3/7 --seed=12345`.
+`node --experimental-strip-types scripts/run-ci-tests.mts 3/14 --seed=12345`.
Refresh timing hints from successful CI JUnit artifacts when shard runtimes
become uneven. From the SDK directory, pass reports for one platform to
@@ -177,37 +178,33 @@ do not. Use `python -m pytest -n 0` to reproduce Python failures serially.
The separate `test-quality` workflow runs weekly, can be dispatched manually,
and runs on pull requests that change its workflow, setup tools, Bun configuration,
shard runner, timing metadata, report comparator, or Windows test-root helper.
-It compares Bun's default runner, `--isolate`, and `--parallel=2` on Linux, macOS,
-and Windows.
-Each mode uses the same duration-based file assignment and production shard count:
-three on Linux/macOS and seven on Windows. This gives 39 sharded runner jobs in place
-of 19 jobs that included an unsharded Windows baseline. It measures the production
-partition rather than cross-file interactions across one unsharded suite; local
-full-suite runs still cover that ordering. More jobs add setup overhead, so compare
-workflow wall time and runner minutes, alongside shard test timings, before
-adopting a runner change.
-
-Every mode uses the same seed for property cases and test ordering: pull requests
+It compares the previous layout (three shards on Linux/macOS, seven on Windows)
+with the expanded production layout (six on Linux/macOS, fourteen on Windows),
+for 39 sharded runner jobs. Both layouts use the same serial Bun runner and
+platform timing snapshot; the comparison combines all shards for each OS before
+checking inventory and outcomes. Local full-suite runs still cover unsharded
+cross-file ordering. More jobs add setup overhead, so compare required-green
+latency and summed runner time alongside shard test timings.
+
+Every layout uses the same seed for property cases and test ordering: pull requests
replay seed 1; scheduled and manual runs use the workflow run number. Reports have
unique OS/mode/shard names. The workflow requires every expected shard report
before accepting comparisons, then compares complete shard groups by test
-identity, outcome, and occurrence count, retaining legitimate repeated names from
-parameterized tests while detecting missing or extra instances. It records every
+identity, outcome, assertion count, and occurrence count, retaining legitimate
+repeated names from parameterized tests while detecting missing or extra instances. It records every
shard's timing, including failed shards, in the job summary. It is not a required
check or part of the release trigger.
-Runner trials use Bun 1.4.3, including the
-[isolated subprocess fix](https://github.com/oven-sh/bun/pull/44581).
-`--parallel=2` runs files in two worker processes and isolates each file; it does
-not make tests inside one file concurrent. Required CI and the mutation trial
-remain on Bun 1.3.14 until a newer release passes the full SDK suite in every mode
-on all three platforms.
-
-Runner modes use the same property seed and coverage settings as each other.
-Required CI uses the default property seed and collects Linux coverage, while
-these diagnostics use the seed above and omit coverage instrumentation. Compare
-inventories across required CI and diagnostics, but measure scheduling speed
-between diagnostic modes to avoid attributing these differences to the runner.
+Required CI and shard diagnostics use Bun 1.3.14 without `--parallel` or
+`--isolate`. Runtime upgrades and worker-based execution need separate
+qualification of process-failure propagation and report/coverage finalization
+before promotion.
+
+Both layouts use the same property seed and coverage settings. Required CI uses
+the default property seed and collects Linux coverage; diagnostics use the seed
+above and the same Linux LCOV instrumentation. Artifacts include JUnit and
+coverage in separate OS/layout/shard paths. Compare inventories across required
+CI and diagnostics, but measure scheduling changes between diagnostic layouts.
Both workflows install ripgrep on Linux so the inventory-path test executes.
Keep the machine-policy test serial. Do not replace the full required suite
diff --git a/sdk/typescript/scripts/compare-test-reports.mts b/sdk/typescript/scripts/compare-test-reports.mts
index 0e44f840e..e75a36c03 100644
--- a/sdk/typescript/scripts/compare-test-reports.mts
+++ b/sdk/typescript/scripts/compare-test-reports.mts
@@ -7,7 +7,11 @@ import { SaxesParser, type SaxesTagNS } from "saxes";
type TestStatus = "passed" | "skipped" | "failed";
type TestIdentity = [file: string, classname: string, name: string];
-type TestOutcome = [...TestIdentity, status: TestStatus];
+type TestOutcome = [
+ ...TestIdentity,
+ status: TestStatus,
+ assertions: string | null,
+];
type TestRecord = {
attributes: SaxesTagNS["attributes"];
status: TestStatus;
@@ -123,7 +127,12 @@ function readReport(path: string) {
attributes["classname"]?.value ?? "",
attributes["name"]?.value ?? "",
];
- const key = JSON.stringify([...identity, status]);
+ const reportedAssertions = attributes["assertions"]?.value;
+ const assertions =
+ reportedAssertions === undefined
+ ? null
+ : integer(reportedAssertions).toString();
+ const key = JSON.stringify([...identity, status, assertions]);
cases.set(key, (cases.get(key) ?? 0) + 1);
}
if (!cases.size) throw new Error(`${path}: no test cases`);
@@ -212,8 +221,8 @@ function main(): number {
.sort(([a], [b]) => {
for (let index = 0; index < a.length; index++) {
const order = Buffer.compare(
- Buffer.from(a[index]!),
- Buffer.from(b[index]!),
+ Buffer.from(a[index] ?? ""),
+ Buffer.from(b[index] ?? ""),
);
if (order) return order;
}
@@ -221,7 +230,10 @@ function main(): number {
});
for (const [identity, count] of differences) {
failed = true;
- console.error(`${label} (${count}): ${identity.join(" > ")}`);
+ const [file, classname, name, status, assertions] = identity;
+ const description = [file, classname, name, status];
+ if (assertions !== null) description.push(`assertions=${assertions}`);
+ console.error(`${label} (${count}): ${description.join(" > ")}`);
}
}
if (failed) return 1;
diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts
index 8e79afa8d..94ae5689c 100644
--- a/sdk/typescript/tests-ts/skeleton.test.ts
+++ b/sdk/typescript/tests-ts/skeleton.test.ts
@@ -139,7 +139,7 @@ describe("TypeScript package skeleton", () => {
const { jobs } = await workflow("node-ci.yml");
expect(jobs["test"]?.strategy?.matrix).toEqual({
os: ["ubuntu-latest", "macos-latest"],
- shard: [1, 2, 3],
+ shard: [1, 2, 3, 4, 5, 6],
});
expect(jobs["compatibility"]?.strategy?.matrix).toEqual({
os: ["ubuntu-latest"],
@@ -147,7 +147,7 @@ describe("TypeScript package skeleton", () => {
include: [{ os: "macos-latest", node: "22.13.0" }],
});
expect(jobs["windows-test"]?.strategy?.matrix).toEqual({
- shard: [1, 2, 3, 4, 5, 6, 7],
+ shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14],
});
expect(jobs["windows-verify"]?.strategy?.matrix["node"]).toEqual([
"22.13.0",
@@ -304,7 +304,7 @@ describe("TypeScript package skeleton", () => {
expect(packageJson.scripts["test:ci"]).toContain("pnpm run test ");
expect(jobs["windows-test"]?.steps).toContainEqual(
expect.objectContaining({
- run: "node --experimental-strip-types sdk/typescript/scripts/run-ci-tests.mts ${{ matrix.shard }}/7 --seed=${{ github.run_number }}",
+ run: "node --experimental-strip-types sdk/typescript/scripts/run-ci-tests.mts ${{ matrix.shard }}/14 --seed=${{ github.run_number }}",
}),
);
});
@@ -467,7 +467,7 @@ describe("TypeScript package skeleton", () => {
expect(restricted).not.toHaveProperty("continue-on-error");
});
- test("keeps machine-wide policy changes out of parallel and experimental runs", async () => {
+ test("keeps machine-wide policy changes out of shared and diagnostic shards", async () => {
const ci = await workflow("node-ci.yml");
const windows = ci.jobs["windows-test"]!.steps!;
expect(
@@ -502,7 +502,7 @@ describe("TypeScript package skeleton", () => {
expect(quality.env?.["CODEX_SECURITY_INTEGRATION"]).toBe("0");
});
- test("keeps runner modes reproducible and report uploads rerunnable", async () => {
+ test("keeps shard layouts reproducible and report uploads rerunnable", async () => {
const ci = await workflow("node-ci.yml");
const quality = await workflow("test-quality.yml");
const runner = quality.jobs["runner"]!;
@@ -516,36 +516,46 @@ describe("TypeScript package skeleton", () => {
"shard"
] as number[];
expect(runner.strategy?.matrix).toEqual({
- target: [
- ...(unixMatrix["os"] as string[]).flatMap((os) =>
- unixShards.map((shard) => ({ os, shard, count: unixShards.length })),
+ target: ["baseline", "expanded"].flatMap((mode) => [
+ ...(unixMatrix["os"] as string[]).flatMap((os) => {
+ const count = mode === "baseline" ? 3 : unixShards.length;
+ return Array.from({ length: count }, (_, index) => ({
+ os,
+ shard: index + 1,
+ count,
+ mode,
+ }));
+ }),
+ ...Array.from(
+ { length: mode === "baseline" ? 7 : windowsShards.length },
+ (_, index) => ({
+ os: "windows-latest",
+ shard: index + 1,
+ count: mode === "baseline" ? 7 : windowsShards.length,
+ mode,
+ }),
),
- ...windowsShards.map((shard) => ({
- os: "windows-latest",
- shard,
- count: windowsShards.length,
- })),
- ],
- mode: [
- { name: "baseline", args: "" },
- { name: "isolated", args: "--isolate" },
- { name: "parallel", args: "--parallel=2" },
- ],
+ ]),
});
- for (const [job, version] of [
- [runner, "1.4.3"],
- [ci.jobs["test"]!, "1.3.14"],
- [ci.jobs["windows-test"]!, "1.3.14"],
- [quality.jobs["mutation"]!, "1.3.14"],
- ] as const) {
+ for (const job of [
+ runner,
+ ci.jobs["test"]!,
+ ci.jobs["windows-test"]!,
+ quality.jobs["mutation"]!,
+ ]) {
expect(
job
.steps!.filter((step) => step.uses?.startsWith("oven-sh/setup-bun@"))
.map((step) => step.with?.["bun-version"]),
- ).toEqual([version]);
+ ).toEqual(["1.3.14"]);
+ for (const step of job.steps!) {
+ expect(step.run ?? "").not.toMatch(
+ /--(?:parallel|isolate)(?:[=\s]|$)/u,
+ );
+ }
}
const execution = runner.steps!.find(
- (step) => step.name === "Test runner mode",
+ (step) => step.name === "Test shard layout",
);
const ripgrep = runner.steps!.find(
(step) => step.name === "Install ripgrep",
@@ -563,7 +573,7 @@ describe("TypeScript package skeleton", () => {
"working-directory": "sdk/typescript",
});
expect(execution?.run).toContain(
- "scripts/run-ci-tests.mts ${{ matrix.target.shard }}/${{ matrix.target.count }} ${{ matrix.mode.args }}",
+ "scripts/run-ci-tests.mts ${{ matrix.target.shard }}/${{ matrix.target.count }}",
);
expect(execution?.run).toContain(
"--seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }}",
@@ -583,7 +593,7 @@ describe("TypeScript package skeleton", () => {
);
expect(runnerUpload).not.toHaveProperty("continue-on-error");
expect(runnerUpload?.with?.["name"]).toBe(
- "runner-${{ matrix.target.os }}-${{ matrix.mode.name }}-${{ matrix.target.shard }}",
+ "runner-${{ matrix.target.os }}-${{ matrix.target.mode }}-${{ matrix.target.shard }}",
);
expect(
quality.jobs["mutation"]?.steps!.find(
diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts
index 74756174b..e03386c24 100644
--- a/sdk/typescript/tests-ts/test-reports.test.ts
+++ b/sdk/typescript/tests-ts/test-reports.test.ts
@@ -44,8 +44,12 @@ async function workflowFixtures() {
runner: {
strategy: {
matrix: {
- target: Array<{ os: string; shard: number }>;
- mode: Array<{ name: string }>;
+ target: Array<{
+ os: string;
+ shard: number;
+ count: number;
+ mode: string;
+ }>;
};
};
};
@@ -53,13 +57,18 @@ async function workflowFixtures() {
};
};
await mkdir(join(fixture.root, "reports"));
- const { target, mode } = workflow.jobs.runner.strategy.matrix;
- for (const { os, shard } of target) {
- for (const { name } of mode) {
- await fixture.report(`reports/runner-${os}-${name}-${shard}.xml`, [
- testcase(`${os} shard ${shard}`),
- ]);
- }
+ const { target } = workflow.jobs.runner.strategy.matrix;
+ for (const { os, shard, count, mode } of target) {
+ const totalCases = Math.max(
+ ...target.filter((item) => item.os === os).map((item) => item.count),
+ );
+ const casesPerShard = totalCases / count;
+ await fixture.report(
+ `reports/runner-${os}-${mode}-${shard}.xml`,
+ Array.from({ length: casesPerShard }, (_, index) =>
+ testcase(`${os} case ${(shard - 1) * casesPerShard + index + 1}`),
+ ),
+ );
}
return {
...fixture,
@@ -69,8 +78,10 @@ async function workflowFixtures() {
};
}
-function testcase(name: string, status = "") {
- return `<testcase file="tests-ts/example.test.ts" classname="example" name="${name}">${status}</testcase>`;
+function testcase(name: string, status = "", assertions?: string) {
+ const assertionAttribute =
+ assertions === undefined ? "" : ` assertions="${assertions}"`;
+ return `<testcase file="tests-ts/example.test.ts" classname="example" name="${name}"${assertionAttribute}>${status}</testcase>`;
}
async function compare(baseline: string, ...candidates: string[]) {
@@ -93,15 +104,9 @@ describe("JUnit inventory comparison", () => {
test("runs every workflow comparison before reporting a mismatch", async () => {
const fixture = await workflowFixtures();
const script = fixture.script;
- const expected = [
- "ubuntu-latest",
- "macos-latest",
- "windows-latest",
- ].flatMap((os) =>
- ["isolated", "parallel"].map(
- (mode) =>
- `reports/runner-${os}-baseline-*.xml|reports/runner-${os}-${mode}-*.xml`,
- ),
+ const expected = ["ubuntu-latest", "macos-latest", "windows-latest"].map(
+ (os) =>
+ `reports/runner-${os}-baseline-*.xml|reports/runner-${os}-expanded-*.xml`,
);
const mock = `node() {
printf '%s\\n' "$3|$4"
@@ -131,34 +136,133 @@ describe("JUnit inventory comparison", () => {
}
});
+ test("requires complete workflow reports even when incomplete unions match", async () => {
+ const fixture = await workflowFixtures();
+ for (const incomplete of [false, true]) {
+ if (incomplete) {
+ // Both unions lose the same two cases, so only completeness catches this.
+ for (const name of ["baseline-3", "expanded-5", "expanded-6"]) {
+ await rm(
+ join(fixture.root, `reports/runner-ubuntu-latest-${name}.xml`),
+ );
+ }
+ }
+ await writeFile(join(fixture.root, "summary.md"), "");
+ const child = Bun.spawn({
+ cmd: [
+ bash,
+ "-e",
+ "-o",
+ "pipefail",
+ "-c",
+ `node() { command node --experimental-strip-types "$CODEX_SECURITY_TEST_COMPARATOR" "$3" "$4"; }\n${fixture.script}`,
+ ],
+ cwd: fixture.root,
+ stdin: "ignore",
+ stdout: "ignore",
+ stderr: "pipe",
+ env: {
+ ...process.env,
+ GITHUB_STEP_SUMMARY: "summary.md",
+ CODEX_SECURITY_TEST_COMPARATOR: join(
+ import.meta.dirname,
+ "..",
+ "scripts",
+ "compare-test-reports.mts",
+ ),
+ },
+ timeout: 10_000,
+ });
+ const { status, stderr } = await readSubprocess(child);
+ expect(status, stderr).toBe(incomplete ? 1 : 0);
+ const summary = await readFile(join(fixture.root, "summary.md"), "utf8");
+ expect(
+ summary.match(/Identical test inventory and outcomes/gu),
+ ).toHaveLength(3);
+ expect(summary).toContain("runner-windows-latest-expanded-14.xml");
+ if (incomplete) {
+ expect(stderr).toContain("Missing shard report:");
+ } else {
+ expect(summary).toContain("runner-ubuntu-latest-expanded-6.xml");
+ }
+ }
+ });
+
for (const scenario of [
- { name: "missing Linux shard", os: "ubuntu-latest", missing: 3 },
- { name: "missing Windows shard", os: "windows-latest", missing: 7 },
{
- name: "missing shard replaced by an unexpected index",
+ name: "missing baseline Linux shard",
+ os: "ubuntu-latest",
+ mode: "baseline",
+ count: 3,
+ missing: 3,
+ },
+ {
+ name: "missing expanded Linux shard",
+ os: "ubuntu-latest",
+ mode: "expanded",
+ count: 6,
+ missing: 6,
+ },
+ {
+ name: "missing baseline Windows shard",
+ os: "windows-latest",
+ mode: "baseline",
+ count: 7,
+ missing: 7,
+ },
+ {
+ name: "missing expanded Windows shard",
+ os: "windows-latest",
+ mode: "expanded",
+ count: 14,
+ missing: 14,
+ },
+ {
+ name: "missing baseline shard replaced by an unexpected index",
os: "ubuntu-latest",
+ mode: "baseline",
+ count: 3,
missing: 3,
extra: 4,
},
- { name: "extra shard", os: "windows-latest", extra: 8 },
+ {
+ name: "missing expanded shard replaced by an unexpected index",
+ os: "ubuntu-latest",
+ mode: "expanded",
+ count: 6,
+ missing: 6,
+ extra: 7,
+ },
+ {
+ name: "extra baseline shard",
+ os: "windows-latest",
+ mode: "baseline",
+ count: 7,
+ extra: 8,
+ },
+ {
+ name: "extra expanded shard",
+ os: "windows-latest",
+ mode: "expanded",
+ count: 14,
+ extra: 15,
+ },
]) {
- test(`rejects identically incomplete workflow groups: ${scenario.name}`, async () => {
+ test(`rejects incomplete workflow groups: ${scenario.name}`, async () => {
const fixture = await workflowFixtures();
- for (const mode of ["baseline", "isolated", "parallel"]) {
- if (scenario.missing !== undefined) {
- await rm(
- join(
- fixture.root,
- `reports/runner-${scenario.os}-${mode}-${scenario.missing}.xml`,
- ),
- );
- }
- if (scenario.extra !== undefined) {
- await fixture.report(
- `reports/runner-${scenario.os}-${mode}-${scenario.extra}.xml`,
- [testcase("unexpected shard")],
- );
- }
+ if (scenario.missing !== undefined) {
+ await rm(
+ join(
+ fixture.root,
+ `reports/runner-${scenario.os}-${scenario.mode}-${scenario.missing}.xml`,
+ ),
+ );
+ }
+ if (scenario.extra !== undefined) {
+ await fixture.report(
+ `reports/runner-${scenario.os}-${scenario.mode}-${scenario.extra}.xml`,
+ [testcase("unexpected shard")],
+ );
}
const child = Bun.spawn({
cmd: [
@@ -180,11 +284,11 @@ describe("JUnit inventory comparison", () => {
expect(status, stderr).toBe(1);
if (scenario.missing !== undefined) {
expect(stderr).toContain(
- `Missing shard report: reports/runner-${scenario.os}-baseline-${scenario.missing}.xml`,
+ `Missing shard report: reports/runner-${scenario.os}-${scenario.mode}-${scenario.missing}.xml`,
);
} else {
expect(stderr).toContain(
- `Expected 7 reports for ${scenario.os}/baseline, found 8`,
+ `Expected ${scenario.count} reports for ${scenario.os}/${scenario.mode}, found ${scenario.count + 1}`,
);
}
// Missing artifacts must not prevent the other OS/mode diagnostics.
@@ -192,7 +296,7 @@ describe("JUnit inventory comparison", () => {
(await readFile(join(fixture.root, "summary.md"), "utf8"))
.trim()
.split(/\r?\n/u),
- ).toHaveLength(6);
+ ).toHaveLength(3);
});
}
@@ -413,6 +517,49 @@ ${testcase("&#97; &#38; &#x22; &#39; &#60; &#62;")}
).toBe(1);
});
+ test("preserves assertion counts for repeated identities across different shard layouts", async () => {
+ const fixture = await fixtures();
+ const repeated = (assertions: string) =>
+ testcase("same parameterized name", "", assertions);
+ await fixture.report("baseline-1.xml", [repeated("2"), repeated("5")]);
+ await fixture.report("baseline-2.xml", [repeated("5")]);
+ await fixture.report("candidate-1.xml", [repeated("5")]);
+ await fixture.report("candidate-2.xml", [repeated("2")]);
+ await fixture.report("candidate-3.xml", [repeated("05")]);
+ const baseline = join(fixture.root, "baseline-*.xml");
+ const candidate = join(fixture.root, "candidate-*.xml");
+ const matching = await compare(baseline, candidate);
+ expect(matching.status, matching.stderr).toBe(0);
+
+ await fixture.report("candidate-1.xml", [repeated("4")]);
+ const changed = await compare(baseline, candidate);
+ expect(changed.status).toBe(1);
+ expect(changed.stderr).toContain(
+ "Missing (1): tests-ts/example.test.ts > example > same parameterized name > passed > assertions=5",
+ );
+ expect(changed.stderr).toContain(
+ "Extra (1): tests-ts/example.test.ts > example > same parameterized name > passed > assertions=4",
+ );
+ });
+
+ test("distinguishes missing assertion counts from reported zero assertions", async () => {
+ const fixture = await fixtures();
+ const baseline = await fixture.report("baseline.xml", [
+ testcase("without assertions", "", "0"),
+ ]);
+ const candidate = await fixture.report("candidate.xml", [
+ testcase("without assertions"),
+ ]);
+ const result = await compare(baseline, candidate);
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain(
+ "Missing (1): tests-ts/example.test.ts > example > without assertions > passed > assertions=0",
+ );
+ expect(result.stderr).toContain(
+ "Extra (1): tests-ts/example.test.ts > example > without assertions > passed",
+ );
+ });
+
test("reports every shard's timing even when an earlier shard fails", async () => {
const fixture = await fixtures();
const first = testcase("first");
diff --git a/sdk/typescript/tests-ts/test-shards.test.ts b/sdk/typescript/tests-ts/test-shards.test.ts
index 8d55cdd5d..d2c77dbff 100644
--- a/sdk/typescript/tests-ts/test-shards.test.ts
+++ b/sdk/typescript/tests-ts/test-shards.test.ts
@@ -65,7 +65,7 @@ test("uses each platform's history and includes new files without selecting dele
expect(partitionTests(files, 2, {}, "darwin").flat().sort()).toEqual(files);
});
-test.each([3, 7])(
+test.each([3, 6, 7, 14])(
"runs each eligible file once across %i native shards, including new files",
async (count) => {
const node = Bun.which("node");
@@ -183,6 +183,51 @@ test("rejects an empty shard while allowing a nonempty shard with the same count
}
});
+test("propagates a subprocess exit after successful test assertions", async () => {
+ const node = Bun.which("node");
+ expect(node).not.toBeNull();
+ const root = await temporaryDirectory("codex-security-shard-exit-");
+ const completed = join(root, "assertion-completed");
+ try {
+ await mkdir(join(root, "scripts"));
+ await mkdir(join(root, "tests-ts"));
+ await copyRunner(root);
+ await writeFile(
+ join(root, "tests-ts", "probe.test.ts"),
+ `import { writeFileSync } from "node:fs";
+import { afterAll, expect, test } from "bun:test";
+test("completed shard assertion", () => {
+ expect(1).toBe(1);
+ writeFileSync(${JSON.stringify(completed)}, "completed");
+});
+afterAll(() => { process.exit(7); });
+`,
+ );
+ const child = Bun.spawn({
+ cmd: [
+ node!,
+ "--experimental-strip-types",
+ join(root, "scripts", "run-ci-tests.mts"),
+ "1/1",
+ ],
+ env: {
+ ...process.env,
+ PATH: `${dirname(process.execPath)}${delimiter}${process.env["PATH"] ?? ""}`,
+ },
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ timeout: 30_000,
+ windowsHide: true,
+ });
+ const { status, stderr } = await readSubprocess(child);
+ expect(await readFile(completed, "utf8")).toBe("completed");
+ expect(status, stderr).toBe(7);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
const defaultTimeoutMs = process.platform === "win32" ? "120000" : "30000";
test.each([