3D engine that AI agents drive
Run CI as parallel jobs with a coverage ratchet and share the checks with publish (issue #560)
Nguyen Xuan An2a746d173277
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.ymlnew file mode 100644index 00000000..0754edc0--- /dev/null+++ b/.github/actions/setup/action.yml@@ -0,0 +1,21 @@+name: Set up 67js+description: Install Node 24, the pinned npm and the locked dependencies. Run actions/checkout first, a local action cannot be found before it.++inputs:+ registry-url:+ description: npm registry for publishing; leave empty for jobs that only test.+ required: false+ default: ''++runs:+ using: composite+ steps:+ - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0+ with:+ node-version: '24'+ registry-url: ${{ inputs.registry-url }}+ package-manager-cache: false+ - run: npm install --global [email protected]+ shell: bash+ - run: npm ci --ignore-scripts+ shell: bashdiff --git a/.github/workflows/publish.yml b/.github/workflows/publish.ymlindex 9b73e800..947fdcc0 100644--- a/.github/workflows/publish.yml+++ b/.github/workflows/publish.yml@@ -8,30 +8,44 @@ on:permissions:contents: read- id-token: writeconcurrency:group: npm-releasecancel-in-progress: falsejobs:+ # The same checks a pull request runs, from test.yml.+ checks:+ uses: ./.github/workflows/test.yml+publish:+ needs: checksruns-on: ubuntu-latest+ permissions:+ contents: read+ id-token: writesteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0+ - uses: ./.github/actions/setupwith:- node-version: '24'registry-url: https://registry.npmjs.org- package-manager-cache: false- - run: npm install --global [email protected]- - run: npm ci --ignore-scripts- - run: npm test- - run: npm run prepare:release- - run: npm run smoke:package- - run: npm run smoke:mcp-package -- --protocol-only- name: Publish a new version through OIDCrun: npm run publish:ci+ - name: Check that the registry serves the version in package.json+ run: |+ expected=$(node -p "require('./package.json').version")+ name=$(node -p "require('./package.json').name")+ for attempt in $(seq 1 12); do+ published=$(npm view "${name}" version 2>/dev/null || true)+ if [ "${published}" = "${expected}" ]; then+ echo "${name}@${published} is live."+ exit 0+ fi+ echo "Registry shows '${published}', waiting for ${expected} (attempt ${attempt} of 12)."+ sleep 10+ done+ echo "The registry did not serve ${name}@${expected} within two minutes." >&2+ exit 1release:needs: publishdiff --git a/.github/workflows/test.yml b/.github/workflows/test.ymlindex dd995e5c..3414c726 100644--- a/.github/workflows/test.yml+++ b/.github/workflows/test.yml@@ -1,30 +1,37 @@name: Test+# Every job runs at once. The last job, named `test`, is the one branch protection requires: it waits for the others and+# fails when any of them did not succeed. publish.yml calls this workflow, so a release runs the same checks as its pull request.on:pull_request:+ workflow_call:permissions:contents: readconcurrency:- group: test-${{ github.event.pull_request.number }}- cancel-in-progress: true+ group: test-${{ github.event.pull_request.number || github.run_id }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}jobs:- test:+ audit:+ runs-on: ubuntu-latest+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - uses: ./.github/actions/setup+ - run: npm audit --audit-level=high++ # Size and tool cost compare against the pull request base, which only a merge commit has, so a release run skips them.+ size:+ if: github.event_name == 'pull_request'runs-on: ubuntu-latest- timeout-minutes: 20+ timeout-minutes: 15steps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:fetch-depth: 2- - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0- with:- node-version: '24'- package-manager-cache: false- - run: npm install --global [email protected]- - run: npm ci --ignore-scripts- - run: npm audit --audit-level=high+ - uses: ./.github/actions/setup# The tested tree is the merge of the pull request into the base branch as it was when the merge was built. Its first parent# is the base of every size, tool cost and archive check, not the newest tip, which may hold work this tree lacks.- name: Find the base of the size checks@@ -39,11 +46,81 @@ jobs:key: size-base-${{ steps.base.outputs.sha }}-${{ hashFiles('scripts/measure-size.mjs', 'scripts/size-*.mjs', 'scripts/*tool-cost*.mjs', 'scripts/measure-archive.mjs', 'scripts/build-mcp.mjs', 'vite.lib.config.js', 'package-lock.json') }}- run: npm run size -- --base "$SIZE_BASE_REF"- run: npm run tool-cost -- --base "$SIZE_BASE_REF"- # The size check packs from its own staging folder, so build/ and build/mcp come from here; the live MCP tests need them.++ # `node --test` already runs every file in its own process (Node 24 default, --test-isolation=process), so one pass over+ # the files is also the isolation check. Three shards keep each runner under a minute of tests.+ unit:+ runs-on: ubuntu-latest+ timeout-minutes: 15+ strategy:+ fail-fast: false+ matrix:+ shard: [1, 2, 3]+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - uses: ./.github/actions/setup+ # The live MCP tests need build/ and build/mcp.- run: npm run build:lib- - run: npm test- - run: npm run test:isolated+ - run: npm run test:coverage -- --test-shard=${{ matrix.shard }}/3 --test-reporter=spec --test-reporter-destination=stdout --test-reporter=lcov --test-reporter-destination=coverage/shard-${{ matrix.shard }}.info+ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2+ if: always()+ with:+ name: lcov-shard-${{ matrix.shard }}+ path: coverage/shard-${{ matrix.shard }}.info+ if-no-files-found: error++ coverage:+ needs: unit+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0+ with:+ node-version: '24'+ package-manager-cache: false+ - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2+ with:+ pattern: lcov-shard-*+ path: coverage/shards+ merge-multiple: true+ - run: node scripts/coverage-gate.mjs coverage/shards --merged-out coverage/lcov.info+ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2+ if: always()+ with:+ name: lcov+ path: coverage/lcov.info++ package:+ runs-on: ubuntu-latest+ timeout-minutes: 15+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - uses: ./.github/actions/setup- run: npm run prepare:release- - run: npm run smoke:package- run: npm run check:links:package+ - run: npm run smoke:package- run: npm run smoke:mcp-package++ test:+ if: always()+ needs: [audit, size, unit, coverage, package]+ runs-on: ubuntu-latest+ steps:+ # A skipped size job is expected outside pull requests. Anything else that is not a success fails the check.+ - name: Fail unless every job succeeded+ env:+ RESULTS: ${{ toJSON(needs) }}+ EVENT_NAME: ${{ github.event_name }}+ run: |+ echo "$RESULTS" | node -e '+ const needs = JSON.parse(require("fs").readFileSync(0, "utf8"));+ let failed = false;+ for (const [name, { result }] of Object.entries(needs)) {+ const expectedSkip = name === "size" && process.env.EVENT_NAME !== "pull_request";+ const ok = result === "success" || (result === "skipped" && expectedSkip);+ console.log(`${ok ? "ok " : "FAIL"} ${name}: ${result}`);+ if (!ok) failed = true;+ }+ process.exit(failed ? 1 : 0);+ 'diff --git a/.gitignore b/.gitignoreindex 6470326e..296956b6 100644--- a/.gitignore+++ b/.gitignore@@ -67,3 +67,5 @@ packages/denoise-oidn/vendor/# Renders and previews that are not part of the packageout/__pycache__/++coverage/diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 1562588e..6fa7276f 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -35,7 +35,7 @@ Run `npm run smoke` or the `smoke:*` script for the area that you changed, and lA change to skinning, glTF loading, poses or IK also needs `npm run smoke:character`, which takes a real character through the MCP server. Run `npm run prepare:release` first.Screenshots and other output go to `artifacts/`, which Git ignores.-Every pull request runs `npm audit`, `npm run size`, `npm run tool-cost`, `npm test`, `npm run test:isolated` (each test file in its own process), `prepare:release`, `smoke:package` and `smoke:mcp-package` in the `Test` workflow. `smoke:mcp-package` draws through the MCP server on SwiftShader in Chrome. `npm run size` compares every bundle with `origin/main`. If a bundle grows past its allowance (64 bytes gzip for core and the scenes, 2% for the npm archive), add `changes/<issue>.size.md` with the line `- entry:core: <why it is worth the bytes>` and say why in the pull request. Never edit `bench/size/budgets.json` or `baseline.json` in a pull request. `bench/size/README.md` has the details.+Every pull request runs `npm audit`, `npm run size`, `npm run tool-cost`, `npm test` (with coverage, three shards; Node runs each file in its own process), `prepare:release`, `smoke:package` and `smoke:mcp-package` in the `Test` workflow. `smoke:mcp-package` draws through the MCP server on SwiftShader in Chrome. `npm run size` compares every bundle with `origin/main`. If a bundle grows past its allowance (64 bytes gzip for core and the scenes, 2% for the npm archive), add `changes/<issue>.size.md` with the line `- entry:core: <why it is worth the bytes>` and say why in the pull request. Never edit `bench/size/budgets.json` or `baseline.json` in a pull request. `bench/size/README.md` has the details.`npm run tool-cost` does the same for what the MCP tools cost an agent. If a tool schema grows by more than 64 bytes, a typical result by more than 128 bytes, or the schema total by more than 128 bytes, first move rare detail into `scene_capabilities` or a shared schema. If the growth is worth it, add a line to `changes/<issue>.size.md`, such as `- tool:scene_create: <why it is worth the bytes>`, `- tool-total: <why>` or `- result:scene_capabilities: <why>`. Never edit `bench/tool-cost-budgets.json` in a pull request. `bench/tool-cost/README.md` has the details.diff --git a/changes/560.md b/changes/560.mdnew file mode 100644index 00000000..91e84c20--- /dev/null+++ b/changes/560.md@@ -0,0 +1 @@+- Split the `Test` workflow into parallel jobs (audit, size and tool cost, three unit shards, coverage, package smokes) behind one final `test` job that branch protection keeps requiring, with a shared `.github/actions/setup`. Unit tests now run with Node's built-in coverage; `scripts/coverage-gate.mjs` merges the shard lcov files, writes the job summary and fails when line, branch or function coverage drops more than 0.5 points below `coverage-baseline.json`. `npm run test:isolated` left CI because `node --test` already runs each file in its own process. `publish.yml` calls `test.yml` before publishing and checks `npm view` afterwards. See `docs/ci.md` (issue #560).diff --git a/coverage-baseline.json b/coverage-baseline.jsonnew file mode 100644index 00000000..c92413e0--- /dev/null+++ b/coverage-baseline.json@@ -0,0 +1 @@+{"lines":93.8,"branches":91.7,"functions":93.9}diff --git a/docs/ci.md b/docs/ci.mdnew file mode 100644index 00000000..8e0ae883--- /dev/null+++ b/docs/ci.md@@ -0,0 +1,28 @@+# CI++`test.yml` runs on every pull request and is called by `publish.yml`, so a release runs the same checks as the pull request that produced it.++## Jobs++All jobs start at once.++| Job | What it runs |+| --- | --- |+| `audit` | `npm audit --audit-level=high` |+| `size` | `npm run size` and `npm run tool-cost` against the pull request base (pull requests only; a release has no base) |+| `unit` (3 shards) | `npm run build:lib`, then `npm run test:coverage` on one third of the test files; uploads its lcov |+| `coverage` | merges the shard lcov files, writes the summary, compares with `coverage-baseline.json`, uploads `lcov` |+| `package` | `prepare:release`, `check:links:package`, `smoke:package`, `smoke:mcp-package` |+| `test` | waits for every job above and fails unless each one succeeded. Branch protection on `main` requires this name. |++`node --test` runs each file in its own process (the Node 24 default, `--test-isolation=process`), so the unit pass is also the isolation check. `npm run test:isolated` stays as a local tool: it runs the same files four at a time and reruns a failure alone.++## Coverage++Coverage comes from Node's built-in `--experimental-test-coverage`. The `coverage` job fails when line, branch or function coverage falls more than 0.5 points below `coverage-baseline.json`. Raise the baseline by editing that file; lower it only on purpose and say why in the pull request. To reproduce locally: `npm run test:coverage -- --test-reporter=lcov --test-reporter-destination=coverage/lcov.info`, then `npm run coverage:gate -- coverage`.++## Add a job or a smoke++1. Add the job to `test.yml`: `actions/checkout`, then `./.github/actions/setup`, then your command. Pin any new action by full SHA.+2. Add its id to `needs` of the `test` job. A job left out of `needs` does not gate `main`.+3. A smoke that needs the packed archive goes in the `package` job after `prepare:release`.diff --git a/package.json b/package.jsonindex f91b355f..35ceb82d 100644--- a/package.json+++ b/package.json@@ -122,6 +122,8 @@"preview": "vite preview --host 127.0.0.1","test": "node --test","test:isolated": "node scripts/test-each-file.mjs",+ "test:coverage": "node --test --experimental-test-coverage --test-coverage-exclude=\"tests/**\" --test-coverage-exclude=\"scripts/**\"",+ "coverage:gate": "node scripts/coverage-gate.mjs","smoke": "node scripts/smoke.mjs","smoke:scenes": "node scripts/smoke-scenes.mjs","smoke:vegetation-impostors": "node scripts/smoke-vegetation-impostors.mjs",diff --git a/scripts/coverage-gate.mjs b/scripts/coverage-gate.mjsnew file mode 100644index 00000000..14006d9d--- /dev/null+++ b/scripts/coverage-gate.mjs@@ -0,0 +1,150 @@+// Merges the lcov files of the test shards, prints and summarises line, branch and function coverage, and fails when+// any figure falls more than the allowed margin below the committed baseline (coverage-baseline.json).+// Raising the baseline is a manual edit of that file.+// Usage: node scripts/coverage-gate.mjs <lcov file or folder>... [--baseline coverage-baseline.json] [--merged-out merged.lcov]+import { appendFileSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';+import { join, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';++const ALLOWED_DROP_POINTS = 0.5;+const METRICS = ['lines', 'branches', 'functions'];++function collectLcovPaths(inputs) {+ const paths = [];+ for (const input of inputs) {+ if (statSync(input).isDirectory()) {+ for (const entry of readdirSync(input, { recursive: true })) {+ const name = String(entry);+ if (name.endsWith('.info') || name.endsWith('.lcov')) paths.push(join(input, name));+ }+ } else {+ paths.push(input);+ }+ }+ return paths;+}++function fileRecord(files, path) {+ if (!files.has(path)) files.set(path, { lines: new Map(), branches: new Map(), functions: new Map() });+ return files.get(path);+}++function addHits(map, key, hits) {+ map.set(key, (map.get(key) ?? 0) + hits);+}++export function mergeLcov(texts) {+ const files = new Map();+ for (const text of texts) {+ let record = null;+ for (const row of text.split(/\r?\n/)) {+ const separator = row.indexOf(':');+ const tag = separator < 0 ? row : row.slice(0, separator);+ const value = separator < 0 ? '' : row.slice(separator + 1);+ if (tag === 'SF') {+ record = fileRecord(files, value);+ } else if (!record) {+ continue;+ } else if (tag === 'DA') {+ const [line, hits] = value.split(',');+ addHits(record.lines, line, Number(hits));+ } else if (tag === 'BRDA') {+ const [line, block, branch, hits] = value.split(',');+ addHits(record.branches, `${line},${block},${branch}`, hits === '-' ? 0 : Number(hits));+ } else if (tag === 'FN') {+ const [line, name] = value.split(',');+ if (!record.functions.has(name)) record.functions.set(name, { line, hits: 0 });+ } else if (tag === 'FNDA') {+ const [hits, name] = value.split(',');+ const entry = record.functions.get(name) ?? { line: '0', hits: 0 };+ entry.hits += Number(hits);+ record.functions.set(name, entry);+ }+ }+ }+ return files;+}++export function totalsOf(files) {+ const totals = { lines: [0, 0], branches: [0, 0], functions: [0, 0] };+ for (const record of files.values()) {+ for (const hits of record.lines.values()) {+ totals.lines[1]++;+ if (hits > 0) totals.lines[0]++;+ }+ for (const hits of record.branches.values()) {+ totals.branches[1]++;+ if (hits > 0) totals.branches[0]++;+ }+ for (const entry of record.functions.values()) {+ totals.functions[1]++;+ if (entry.hits > 0) totals.functions[0]++;+ }+ }+ const percentages = {};+ for (const metric of METRICS) {+ const [covered, found] = totals[metric];+ percentages[metric] = found === 0 ? 100 : Math.round((covered / found) * 10000) / 100;+ }+ return { totals, percentages };+}++export function compareWithBaseline(percentages, baseline) {+ return METRICS.map(metric => ({+ metric,+ actual: percentages[metric],+ baseline: baseline[metric],+ ok: percentages[metric] >= baseline[metric] - ALLOWED_DROP_POINTS,+ }));+}++function toLcov(files) {+ const rows = [];+ for (const [path, record] of files) {+ rows.push('TN:', `SF:${path}`);+ for (const [name, entry] of record.functions) rows.push(`FN:${entry.line},${name}`);+ for (const [name, entry] of record.functions) rows.push(`FNDA:${entry.hits},${name}`);+ for (const [key, hits] of record.branches) rows.push(`BRDA:${key},${hits}`);+ for (const [line, hits] of record.lines) rows.push(`DA:${line},${hits}`);+ rows.push('end_of_record');+ }+ return rows.join('\n') + '\n';+}++function main() {+ const args = process.argv.slice(2);+ const inputs = [];+ let baselinePath = 'coverage-baseline.json';+ let mergedOut = null;+ for (let index = 0; index < args.length; index++) {+ if (args[index] === '--baseline') baselinePath = args[++index];+ else if (args[index] === '--merged-out') mergedOut = args[++index];+ else inputs.push(args[index]);+ }+ const paths = collectLcovPaths(inputs);+ if (paths.length === 0) throw new Error('No lcov files found.');+ const files = mergeLcov(paths.map(path => readFileSync(path, 'utf8')));+ if (mergedOut) writeFileSync(mergedOut, toLcov(files));+ const { totals, percentages } = totalsOf(files);+ const baseline = JSON.parse(readFileSync(baselinePath, 'utf8'));+ const results = compareWithBaseline(percentages, baseline);++ const table = [+ `Coverage from ${paths.length} lcov file(s), ${files.size} source files. A drop of more than ${ALLOWED_DROP_POINTS} points below the baseline fails.`,+ '',+ '| Metric | Covered | Total | Actual % | Baseline % | Result |',+ '| --- | ---: | ---: | ---: | ---: | --- |',+ ...results.map(+ result => `| ${result.metric} | ${totals[result.metric][0]} | ${totals[result.metric][1]} | ${result.actual} | ${result.baseline} | ${result.ok ? 'pass' : 'FAIL'} |`,+ ),+ '',+ ].join('\n');+ console.log(table);+ if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### Coverage\n\n${table}\n`);+ if (results.some(result => !result.ok)) {+ console.error('Coverage fell below the baseline. Add tests, or lower coverage-baseline.json on purpose.');+ process.exit(1);+ }+}++if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main();