Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

ci(web): cut web PR check time without changing what they check (#9949)

teocd9f3d506319
diff --git a/.github/workflows/assign-labels.yml b/.github/workflows/assign-labels.yml
index 0171e4b43b..0b47a725bd 100644
--- a/.github/workflows/assign-labels.yml
+++ b/.github/workflows/assign-labels.yml
@@ -14,28 +14,23 @@ jobs:
contents: read
pull-requests: write
steps:
- - name: Checkout
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- with:
- clean: 'false'
- fetch-depth: '0'
- persist-credentials: 'false'
- - id: changed-files
- name: Get changed files
- uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62
- with:
- json: 'true'
- escape_json: 'false'
- write_output_files: 'true'
- name: Label based on paths
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
- const fs = require('fs');
- const changedFiles = JSON.parse(
- fs.readFileSync('.github/outputs/all_changed_files.json', 'utf8')
- );
+ // The PR's files from the API, without the files it deletes: the
+ // same set a full-history checkout and diff gave, in about 4 s
+ // instead of 38 s (once 705 s). The API lists at most 3,000 files.
+ const files = await github.paginate(github.rest.pulls.listFiles, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: context.issue.number,
+ per_page: 100,
+ });
+ const changedFiles = files
+ .filter((file) => file.status !== 'removed')
+ .map((file) => file.filename);
const labels = new Set();
// Define path-to-label mappings
diff --git a/.github/workflows/web-app-check-main.yml b/.github/workflows/web-app-check-main.yml
index d9e6de8945..b2fda2815f 100644
--- a/.github/workflows/web-app-check-main.yml
+++ b/.github/workflows/web-app-check-main.yml
@@ -16,6 +16,9 @@ jobs:
steps:
- name: Checkout Repo
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
+ with:
+ filter: blob:none
+ sparse-checkout: '.github'
- id: filter
name: Filter changed paths
uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590
@@ -98,19 +101,21 @@ jobs:
[ -n "$base" ] || result "No base commit to compare with." true
git fetch -q --depth=1 origin "$base"
names='utoipa|ToSchema|IntoParams|specta|schemars|JsonSchema'
- # Every grep here reads all its input. One that stopped at its first
- # match would close the pipe, and pipefail reads the writer's SIGPIPE as
- # no match.
- if git diff "$base" HEAD -- Cargo.toml Cargo.lock | grep -E "^[-+].*($names)"; then
+ manifests=$(git diff "$base" HEAD -- Cargo.toml Cargo.lock)
+ if grep -E "^[-+].*($names)" <<< "$manifests"; then
result "A schema crate's version changed." true
fi
+ changed=$(git diff --name-only --no-renames "$base" HEAD -- 'crates/*.rs' 'services/*.rs')
while IFS= read -r file; do
+ [ -n "$file" ] || continue
for revision in HEAD "$base"; do
- if git show "$revision:$file" 2>/dev/null | grep -E "$names" > /dev/null; then
+ [ -n "$(git ls-tree "$revision" -- "$file")" ] || continue
+ contents=$(git show "$revision:$file")
+ if grep -E "$names" <<< "$contents" > /dev/null; then
result "$file names a schema item." true
fi
done
- done < <(git diff --name-only --no-renames "$base" HEAD -- 'crates/*.rs' 'services/*.rs')
+ done <<< "$changed"
result "No changed Rust file names a schema item." false
env:
CODEGEN_INPUTS: ${{ steps.filter.outputs.codegen_inputs }}
@@ -333,37 +338,48 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup Prereqs
uses: './.github/actions/setup-reqs-web'
- - name: Check Types
- run: bun run type-check
- working-directory: apps/web
- - name: Check Floating Promises
- run: bun run lint:oxlint --quiet
- - name: Check Functional Test Types
- run: bun ../../node_modules/typescript-native/bin/tsc --noEmit --skipLibCheck --project tests/functional/tsconfig.json
- working-directory: apps/web
- - name: Check Collaboration Package Types
- run: bun run type-check
- working-directory: packages/collaboration
- - name: Check Lexical Service Types
- run: bun run check
- working-directory: services/lexical-service
- - name: Test Lexical Service Endpoints
- run: bun test src
- working-directory: services/lexical-service
- name: Install SDK Documentation Dependencies
run: bun install --frozen-lockfile
working-directory: packages/sdk
- - name: Check SDK Documentation
- run: bun run type-check && bun run test
- working-directory: packages/sdk-docs
- - name: Check Editing Worker
- run: bun run type-check
- working-directory: services/ai-editing-worker
- - name: Check Code Mode SDK
- run: bun run type-check && bun run test
- working-directory: packages/code-mode-sdk
+ - name: Run checks
+ run: |
+ set -uo pipefail
+ logs="$RUNNER_TEMP/typecheck"
+ mkdir -p "$logs"
+ names=()
+ pids=()
+ check() {
+ local name="$1" directory="$2" command="$3"
+ ( cd "$directory" && eval "$command" ) > "$logs/${#names[@]}.log" 2>&1 &
+ names+=("$name")
+ pids+=("$!")
+ }
+ check 'Check Types' 'apps/web' 'bun run type-check'
+ check 'Check Floating Promises' '.' 'bun run lint:oxlint --quiet'
+ check 'Check Functional Test Types' 'apps/web' 'bun ../../node_modules/typescript-native/bin/tsc --noEmit --skipLibCheck --project tests/functional/tsconfig.json'
+ check 'Check Collaboration Package Types' 'packages/collaboration' 'bun run type-check'
+ check 'Check Lexical Service Types' 'services/lexical-service' 'bun run check'
+ check 'Test Lexical Service Endpoints' 'services/lexical-service' 'bun test src'
+ check 'Check SDK Documentation' 'packages/sdk-docs' 'bun run type-check && bun run test'
+ check 'Check Editing Worker' 'services/ai-editing-worker' 'bun run type-check'
+ check 'Check Code Mode SDK' 'packages/code-mode-sdk' 'bun run type-check && bun run test'
+ failed=0
+ for index in "${!names[@]}"; do
+ status=0
+ wait "${pids[$index]}" || status=$?
+ echo "::group::${names[$index]} (exit $status)"
+ cat "$logs/$index.log"
+ echo "::endgroup::"
+ if [ "$status" -ne 0 ]; then
+ echo "::error::${names[$index]} failed with exit code $status"
+ failed=1
+ fi
+ done
+ exit "$failed"
- name: Teardown Nix
if: always()
uses: './.github/actions/teardown-nix'
@@ -563,6 +579,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
- name: Run Biome
@@ -580,7 +598,7 @@ jobs:
needs:
- path-check
if: needs.path-check.outputs.should_run == 'true'
- name: Test (${{ matrix.shard }}/3)
+ name: Test (${{ matrix.shard }}/6)
runs-on: namespace-profile-linux-mid;overrides.cache-tag=web-ci
strategy:
matrix:
@@ -588,6 +606,9 @@ jobs:
- 1
- 2
- 3
+ - 4
+ - 5
+ - 6
fail-fast: false
steps:
- name: Checkout Repo
@@ -606,6 +627,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup
uses: './.github/actions/setup-reqs-web'
- name: Configure Namespace remote sccache
@@ -648,7 +671,7 @@ jobs:
timeout-minutes: 15
working-directory: apps/web
- name: Test
- run: bun run test --shard=${{ matrix.shard }}/3
+ run: bun run test --shard=${{ matrix.shard }}/6
env:
VITEST_MODULE_CACHE_DIR: /home/runner/.cache/vitest-modules
working-directory: apps/web
@@ -677,6 +700,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup
uses: './.github/actions/setup-reqs-web'
with:
@@ -761,6 +786,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup
uses: './.github/actions/setup-reqs-web'
with:
@@ -844,6 +871,8 @@ jobs:
continue-on-error: true
- name: Setup Nix
uses: './.github/actions/setup-nix'
+ with:
+ seed-cache: sccache-ci
- name: Setup Nix dev shell
uses: './.github/actions/setup-nix-dev-shell'
- name: Cycles Import Check
diff --git a/apps/web/scripts/cached-wasm-build.sh b/apps/web/scripts/cached-wasm-build.sh
index 0228c519f9..70194178a1 100755
--- a/apps/web/scripts/cached-wasm-build.sh
+++ b/apps/web/scripts/cached-wasm-build.sh
@@ -1,11 +1,14 @@
#!/usr/bin/env bash
-# Runs a wasm package build, or restores its output from WASM_PKG_CACHE_DIR when
-# nothing the package is built from has changed. Without WASM_PKG_CACHE_DIR it
-# only runs the build, so local `just build-*` behaves as before.
+# Runs a wasm package build, or restores its output when nothing the package is
+# built from has changed: first from WASM_PKG_CACHE_DIR on the node's cache
+# volume, then from Namespace's HTTP cache, which every node reads. Without
+# WASM_PKG_CACHE_DIR it only runs the build, so local `just build-*` behaves as
+# before. Only PR check jobs set it; deploys always build.
#
-# The cache key hashes the contents of the crate's workspace dependency closure
-# (.github/workspace-dep-closures.json), the root manifests and lockfile, the
-# toolchain pins, the shared assets build scripts read, and the build command.
+# wasm-cache-key.ts computes the key from the package's workspace dependency
+# closure (.github/workspace-dep-closures.json), its slice of Cargo.lock, the
+# root manifest, the toolchain and Nix definitions, the shared assets build
+# scripts read, its profile's env and the build command.
#
# usage: cached-wasm-build.sh <closure package> <out dir> -- <build command...>
set -euo pipefail
@@ -21,38 +24,65 @@ if [ -z "${WASM_PKG_CACHE_DIR:-}" ]; then
exec "$@"
fi
-root="$(git rev-parse --show-toplevel)"
-mapfile -t dirs < <(jq -r --arg p "$package" '.closures[$p] // empty | .[]' \
- "$root/.github/workspace-dep-closures.json")
-if [ "${#dirs[@]}" -eq 0 ]; then
- echo "no dependency closure for $package; building without the cache" >&2
- exec "$@"
-fi
-
-key="$(
- {
- (cd "$root" && git ls-files -z -- "${dirs[@]}" Cargo.toml Cargo.lock \
- rust-toolchain.toml flake.lock .cargo static_assets | xargs -0 sha256sum)
- printf '%s\n' "$@"
- } | sha256sum | cut -c1-32
-)"
+key="$(bun "$(dirname "$0")/wasm-cache-key.ts" "$package" -- "$@")"
entry="$WASM_PKG_CACHE_DIR/$package-$key"
-if [ -d "$entry" ]; then
- echo "$package: unchanged inputs, reusing the cached wasm package ($key)"
+restore() {
rm -rf "$out_dir"
mkdir -p "$(dirname "$out_dir")"
- cp -a "$entry" "$out_dir"
+ cp -a "$1" "$out_dir"
+}
+
+# Keeps a copy on this node's volume. A concurrent job may have stored the same
+# key first; that copy wins.
+store_locally() {
+ mkdir -p "$WASM_PKG_CACHE_DIR"
+ # Entries are small and keyed by content; drop ones no build has used for a week.
+ find "$WASM_PKG_CACHE_DIR" -mindepth 1 -maxdepth 1 -mtime +7 -exec rm -rf {} + || true
+ local tmp
+ tmp="$(mktemp -d "$WASM_PKG_CACHE_DIR/.tmp-XXXXXX")"
+ cp -a "$1/." "$tmp/"
+ mv -T "$tmp" "$entry" 2>/dev/null || rm -rf "$tmp"
+}
+
+if [ -d "$entry" ]; then
+ echo "$package: unchanged inputs, reusing the cached wasm package ($key)"
+ restore "$entry"
touch "$entry"
exit 0
fi
+# The HTTP cache uses the remote sccache credentials the job configured; forks
+# have none. PR jobs write it, as they write the cache volume.
+remote=""
+if [ -n "${SCCACHE_WEBDAV_ENDPOINT:-}" ] && [ -n "${SCCACHE_WEBDAV_TOKEN:-}" ]; then
+ remote="$SCCACHE_WEBDAV_ENDPOINT/wasm-pkg/$package-$key.tar.zst"
+fi
+
+# The token goes through a config descriptor so it stays out of curl's arguments.
+remote_request() {
+ curl --fail --silent --show-error --max-time 60 \
+ -K <(printf 'header = "Authorization: Bearer %s"\n' "$SCCACHE_WEBDAV_TOKEN") "$@"
+}
+
+if [ -n "$remote" ]; then
+ fetched="$(mktemp -d)"
+ if remote_request "$remote" 2>/dev/null | zstd -d -q 2>/dev/null | tar -xf - -C "$fetched" 2>/dev/null \
+ && [ -n "$(ls -A "$fetched")" ]; then
+ echo "$package: unchanged inputs, restored the wasm package from the HTTP cache ($key)"
+ restore "$fetched"
+ store_locally "$fetched"
+ rm -rf "$fetched"
+ exit 0
+ fi
+ rm -rf "$fetched"
+ echo "$package: not cached, building ($key)"
+fi
+
"$@"
-mkdir -p "$WASM_PKG_CACHE_DIR"
-# Entries are small and keyed by content; drop ones no build has used for a week.
-find "$WASM_PKG_CACHE_DIR" -mindepth 1 -maxdepth 1 -mtime +7 -exec rm -rf {} + || true
-tmp="$(mktemp -d "$WASM_PKG_CACHE_DIR/.tmp-XXXXXX")"
-cp -a "$out_dir/." "$tmp/"
-# A concurrent job may have stored the same key first; keep that one.
-mv -T "$tmp" "$entry" 2>/dev/null || rm -rf "$tmp"
+if [ -n "$remote" ]; then
+ tar -C "$out_dir" -cf - . | zstd -q -3 -T0 | remote_request --upload-file - "$remote" \
+ || echo "::warning::$package: could not store the wasm package in the HTTP cache"
+fi
+store_locally "$out_dir"
diff --git a/apps/web/scripts/wasm-cache-key.test.ts b/apps/web/scripts/wasm-cache-key.test.ts
new file mode 100644
index 0000000000..1556f6958d
--- /dev/null
+++ b/apps/web/scripts/wasm-cache-key.test.ts
@@ -0,0 +1,166 @@
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import {
+ lockSlice,
+ manifestWithoutMembers,
+ profileEnvPrefix,
+ wasmCacheKey,
+} from './wasm-cache-key';
+
+const LOCK = `# This file is automatically @generated by Cargo.
+version = 4
+
+[[package]]
+name = "engine"
+version = "0.1.0"
+dependencies = [
+ "png",
+ "serde 1.0.200",
+]
+
+[[package]]
+name = "png"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aaaa"
+
+[[package]]
+name = "serde"
+version = "1.0.200"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbbb"
+
+[[package]]
+name = "serde"
+version = "1.0.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccc"
+
+[[package]]
+name = "server"
+version = "0.1.0"
+dependencies = ["serde 1.0.100", "tokio"]
+
+[[package]]
+name = "tokio"
+version = "1.40.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dddd"
+`;
+
+const MANIFEST = `[workspace]
+resolver = "2"
+members = [
+ "crates/engine",
+ "crates/server",
+]
+
+[workspace.dependencies]
+png = "0.17"
+
+[profile.engine-wasm]
+inherits = "release"
+lto = true
+`;
+
+const temporaryDirectories: string[] = [];
+afterEach(() => {
+ for (const directory of temporaryDirectories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true });
+ }
+});
+
+function workspace(files: Record<string, string>): string {
+ const root = mkdtempSync(join(tmpdir(), 'wasm-cache-key-'));
+ temporaryDirectories.push(root);
+ for (const [path, contents] of Object.entries(files)) {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), contents);
+ }
+ symlinkSync('../../AGENTS.md', join(root, 'crates/engine/CLAUDE.md'));
+ spawnSync('git', ['init', '-q'], { cwd: root });
+ spawnSync('git', ['add', '-A'], { cwd: root });
+ return root;
+}
+
+const BASE_FILES = {
+ 'Cargo.toml': MANIFEST,
+ 'Cargo.lock': LOCK,
+ 'rust-toolchain.toml': '[toolchain]\nchannel = "1.94.0"\n',
+ 'nix/tools.nix': '{ wasm-pack = "0.15.0"; }\n',
+ '.github/workspace-dep-closures.json': JSON.stringify({
+ closures: { engine: ['crates/engine'], server: ['crates/server'] },
+ }),
+ 'crates/engine/Cargo.toml': '[package]\nname = "engine"\nversion = "0.1.0"\n',
+ 'crates/engine/src/lib.rs': 'pub fn draw() {}\n',
+ 'crates/server/Cargo.toml': '[package]\nname = "server"\nversion = "0.1.0"\n',
+ 'crates/server/src/main.rs': 'fn main() {}\n',
+};
+
+const BUILD = ['wasm-pack', 'build', 'crates/engine', '--profile', 'engine-wasm'];
+
+function key(files: Record<string, string>, env: Record<string, string> = {}) {
+ return wasmCacheKey(workspace({ ...BASE_FILES, ...files }), 'engine', BUILD, env);
+}
+
+describe('lockSlice', () => {
+ it('follows locked dependencies, choosing the named version', () => {
+ const slice = lockSlice(LOCK, ['engine']).join('\n');
+ expect(slice).toContain('name = "png"');
+ expect(slice).toContain('checksum = "bbbb"');
+ expect(slice).not.toContain('checksum = "cccc"');
+ expect(slice).not.toContain('name = "tokio"');
+ });
+});
+
+describe('manifestWithoutMembers', () => {
+ it('drops only the member list', () => {
+ const manifest = manifestWithoutMembers(MANIFEST);
+ expect(manifest).not.toContain('crates/server');
+ expect(manifest).toContain('resolver = "2"');
+ expect(manifest).toContain('png = "0.17"');
+ expect(manifest).toContain('[profile.engine-wasm]');
+ });
+});
+
+describe('profileEnvPrefix', () => {
+ it('reads --profile, and release otherwise', () => {
+ expect(profileEnvPrefix(BUILD)).toBe('CARGO_PROFILE_ENGINE_WASM_');
+ expect(profileEnvPrefix(['wasm-pack', 'build', '--release'])).toBe(
+ 'CARGO_PROFILE_RELEASE_'
+ );
+ });
+});
+
+describe('wasmCacheKey', () => {
+ it('keeps the key when only packages outside the build change', () => {
+ const base = key({});
+ expect(key({ 'crates/server/src/main.rs': 'fn main() { run() }\n' })).toBe(base);
+ expect(
+ key({ 'Cargo.lock': LOCK.replace('version = "1.40.0"', 'version = "1.41.0"') })
+ ).toBe(base);
+ expect(
+ key({ 'Cargo.toml': MANIFEST.replace('"crates/server",', '"crates/server",\n "crates/new",') })
+ ).toBe(base);
+ expect(key({}, { CARGO_PROFILE_OTHER_WASM_LTO: 'off' })).toBe(base);
+ });
+
+ it('changes the key when anything the build reads changes', () => {
+ const base = key({});
+ const changed = [
+ key({ 'crates/engine/src/lib.rs': 'pub fn draw() { paint() }\n' }),
+ key({ 'Cargo.lock': LOCK.replace('checksum = "aaaa"', 'checksum = "eeee"') }),
+ key({ 'Cargo.lock': LOCK.replace('checksum = "bbbb"', 'checksum = "ffff"') }),
+ key({ 'Cargo.toml': MANIFEST.replace('png = "0.17"', 'png = "0.18"') }),
+ key({ 'Cargo.toml': MANIFEST.replace('lto = true', 'lto = false') }),
+ key({ 'rust-toolchain.toml': '[toolchain]\nchannel = "1.95.0"\n' }),
+ key({ 'nix/tools.nix': '{ wasm-pack = "0.15.1"; }\n' }),
+ key({}, { CARGO_PROFILE_ENGINE_WASM_LTO: 'off' }),
+ wasmCacheKey(workspace(BASE_FILES), 'engine', [...BUILD, '--target', 'web'], {}),
+ ];
+ for (const other of changed) expect(other).not.toBe(base);
+ });
+});
diff --git a/apps/web/scripts/wasm-cache-key.ts b/apps/web/scripts/wasm-cache-key.ts
new file mode 100644
index 0000000000..bd42a56776
--- /dev/null
+++ b/apps/web/scripts/wasm-cache-key.ts
@@ -0,0 +1,239 @@
+// Prints the cache key for one wasm package build: a hash of what the build
+// reads, so a change elsewhere in the workspace keeps the key.
+//
+// - The files of the package's workspace dependency closure
+// (.github/workspace-dep-closures.json), the toolchain pin, the Nix flake and
+// its definitions (they choose wasm-pack, wasm-bindgen and binaryen), `.cargo`
+// and `static_assets`.
+// - Its slice of Cargo.lock: every locked package reachable from the closure's
+// crates. Lockfile edges cover every target and dependency kind, so the slice
+// holds at least what the build compiles.
+// - The root Cargo.toml without its `members` list. Dependencies, profiles,
+// patches and lints stay in.
+// - The `CARGO_PROFILE_*` variables of the profile the build uses, and the
+// build command.
+//
+// usage: bun scripts/wasm-cache-key.ts <closure package> -- <build command...>
+
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { lstatSync, readFileSync, readlinkSync } from 'node:fs';
+import { join } from 'node:path';
+
+type LockPackage = {
+ name: string;
+ version: string;
+ lines: string[];
+ dependencies: string[];
+};
+
+/** Cargo.lock's `[[package]]` entries, read from the format cargo writes. */
+function parseLock(lockText: string): LockPackage[] {
+ const packages: LockPackage[] = [];
+ let current: LockPackage | undefined;
+ let inDependencies = false;
+
+ for (const line of lockText.split('\n')) {
+ if (line === '[[package]]') {
+ current = { name: '', version: '', lines: [], dependencies: [] };
+ packages.push(current);
+ inDependencies = false;
+ continue;
+ }
+ if (current === undefined || line.trim() === '') continue;
+
+ current.lines.push(line);
+ if (inDependencies) {
+ if (line === ']') {
+ inDependencies = false;
+ } else {
+ current.dependencies.push(line.trim().replace(/^"|",?$/g, ''));
+ }
+ continue;
+ }
+
+ const field = /^(\w+) = (.*)$/.exec(line);
+ if (field === null) continue;
+ const [, key, value] = field;
+ if (key === 'name') current.name = JSON.parse(value);
+ if (key === 'version') current.version = JSON.parse(value);
+ if (key === 'dependencies') {
+ if (value === '[') {
+ inDependencies = true;
+ } else {
+ current.dependencies.push(...(JSON.parse(value) as string[]));
+ }
+ }
+ }
+
+ return packages;
+}
+
+/**
+ * The Cargo.lock entries reachable from `roots` (package names), as text in a
+ * fixed order.
+ */
+export function lockSlice(lockText: string, roots: string[]): string[] {
+ const byName = new Map<string, LockPackage[]>();
+ for (const lockPackage of parseLock(lockText)) {
+ const sameName = byName.get(lockPackage.name) ?? [];
+ sameName.push(lockPackage);
+ byName.set(lockPackage.name, sameName);
+ }
+
+ // A dependency reads `name`, `name version` or `name version (source)`; the
+ // version appears only when more than one version of that name is locked.
+ const resolveDependency = (dependency: string): LockPackage[] => {
+ const [name, version] = dependency.split(' ');
+ const sameName = byName.get(name) ?? [];
+ if (version === undefined) return sameName;
+
+ return sameName.filter((lockPackage) => lockPackage.version === version);
+ };
+
+ const reached = new Set<LockPackage>();
+ const pending = roots.flatMap((name) => byName.get(name) ?? []);
+ for (let next = pending.pop(); next !== undefined; next = pending.pop()) {
+ if (reached.has(next)) continue;
+
+ reached.add(next);
+ for (const dependency of next.dependencies) {
+ pending.push(...resolveDependency(dependency));
+ }
+ }
+
+ return [...reached].map((lockPackage) => lockPackage.lines.join('\n')).sort();
+}
+
+/** The root manifest without the `[workspace]` table's `members` array. */
+export function manifestWithoutMembers(manifestText: string): string {
+ const kept: string[] = [];
+ let section = '';
+ let inMembers = false;
+
+ for (const line of manifestText.split('\n')) {
+ if (inMembers) {
+ if (line.trim() === ']') inMembers = false;
+ continue;
+ }
+
+ const header = /^\[([^\]]+)\]\s*$/.exec(line);
+ if (header !== null) section = header[1];
+
+ if (section === 'workspace' && /^members\s*=/.test(line)) {
+ inMembers = !line.includes(']');
+ continue;
+ }
+
+ kept.push(line);
+ }
+
+ return kept.join('\n');
+}
+
+/** The env var prefix cargo reads for the profile a build command selects. */
+export function profileEnvPrefix(command: string[]): string {
+ const flag = command.indexOf('--profile');
+ const profile = flag === -1 ? 'release' : command[flag + 1];
+ return `CARGO_PROFILE_${profile.toUpperCase().replaceAll('-', '_')}_`;
+}
+
+function git(directory: string, args: string[]): string {
+ const result = spawnSync('git', args, { cwd: directory, encoding: 'utf8' });
+ if (result.status !== 0) {
+ throw new Error(`git ${args.join(' ')} failed: ${result.stderr}`);
+ }
+
+ return result.stdout;
+}
+
+/** The package names declared by the closure's workspace crates. */
+function crateNames(root: string, directories: string[]): string[] {
+ return directories.map((directory) => {
+ const manifest = readFileSync(join(root, directory, 'Cargo.toml'), 'utf8');
+ let section = '';
+ for (const line of manifest.split('\n')) {
+ const header = /^\[([^\]]+)\]\s*$/.exec(line);
+ if (header !== null) section = header[1];
+
+ const name = /^name\s*=\s*"([^"]+)"/.exec(line);
+ if (section === 'package' && name !== null) return name[1];
+ }
+
+ throw new Error(`no package name in ${directory}/Cargo.toml`);
+ });
+}
+
+export function wasmCacheKey(
+ root: string,
+ closurePackage: string,
+ command: string[],
+ env: Record<string, string | undefined>
+): string {
+ const closures = JSON.parse(
+ readFileSync(join(root, '.github/workspace-dep-closures.json'), 'utf8')
+ ) as { closures: Record<string, string[]> };
+ const directories = closures.closures[closurePackage] ?? [];
+ if (directories.length === 0) {
+ throw new Error(`no dependency closure for ${closurePackage}`);
+ }
+
+ const hash = createHash('sha256');
+
+ const files = git(root, [
+ 'ls-files',
+ '-z',
+ '--',
+ ...directories,
+ 'rust-toolchain.toml',
+ 'flake.lock',
+ 'flake.nix',
+ 'nix',
+ '.cargo',
+ 'static_assets',
+ ])
+ .split('\0')
+ .filter((file) => file !== '');
+ for (const file of files) {
+ // Symlinks (the CLAUDE.md links) count by target, as git stores them.
+ const path = join(root, file);
+ const isLink = lstatSync(path).isSymbolicLink();
+ hash.update(`${isLink ? 'link' : 'file'} ${file}\0`);
+ hash.update(isLink ? readlinkSync(path) : readFileSync(path));
+ hash.update('\0');
+ }
+
+ const lock = lockSlice(
+ readFileSync(join(root, 'Cargo.lock'), 'utf8'),
+ crateNames(root, directories)
+ );
+ hash.update(`lock\0${lock.join('\0')}\0`);
+
+ const manifest = manifestWithoutMembers(
+ readFileSync(join(root, 'Cargo.toml'), 'utf8')
+ );
+ hash.update(`manifest\0${manifest}\0`);
+
+ const prefix = profileEnvPrefix(command);
+ const profileEnv = Object.entries(env)
+ .filter(([name]) => name.startsWith(prefix))
+ .map(([name, value]) => `${name}=${value ?? ''}`)
+ .sort();
+ hash.update(`env\0${profileEnv.join('\0')}\0`);
+ hash.update(`command\0${command.join('\0')}\0`);
+
+ return hash.digest('hex').slice(0, 32);
+}
+
+if (import.meta.main) {
+ const [closurePackage, separator, ...command] = process.argv.slice(2);
+ if (closurePackage === undefined || separator !== '--' || command.length === 0) {
+ console.error(
+ 'usage: bun scripts/wasm-cache-key.ts <closure package> -- <build command...>'
+ );
+ process.exit(2);
+ }
+
+ const root = git(process.cwd(), ['rev-parse', '--show-toplevel']).trim();
+ console.log(wasmCacheKey(root, closurePackage, command, process.env));
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/assign_labels.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/assign_labels.rs
index df5985ef22..90e7a08076 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/assign_labels.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/assign_labels.rs
@@ -2,7 +2,7 @@
use gh_workflow::{Event, Job, Level, Permissions, PullRequest, PullRequestType, Step, Workflow};
-use crate::workflows::{runners, steps};
+use crate::workflows::runners;
/// Build the workflow.
pub fn assign_labels() -> Workflow {
@@ -24,24 +24,9 @@ fn label_job() -> Job {
pull_requests: Some(Level::Write),
..Default::default()
})
- .add_step(steps::checkout(true, false))
- .add_step(changed_files())
.add_step(label_by_paths())
}
-fn changed_files() -> Step<gh_workflow::Use> {
- Step::new("Get changed files")
- .uses(
- "tj-actions",
- "changed-files",
- "24d32ffd492484c1d75e0c0b894501ddb9d30d62",
- )
- .id("changed-files")
- .add_with(("json", true))
- .add_with(("escape_json", false))
- .add_with(("write_output_files", true))
-}
-
fn label_by_paths() -> Step<gh_workflow::Use> {
Step::new("Label based on paths")
.uses(
@@ -53,10 +38,18 @@ fn label_by_paths() -> Step<gh_workflow::Use> {
.add_with((
"script",
indoc::indoc! {r#"
- const fs = require('fs');
- const changedFiles = JSON.parse(
- fs.readFileSync('.github/outputs/all_changed_files.json', 'utf8')
- );
+ // The PR's files from the API, without the files it deletes: the
+ // same set a full-history checkout and diff gave, in about 4 s
+ // instead of 38 s (once 705 s). The API lists at most 3,000 files.
+ const files = await github.paginate(github.rest.pulls.listFiles, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: context.issue.number,
+ per_page: 100,
+ });
+ const changedFiles = files
+ .filter((file) => file.status !== 'removed')
+ .map((file) => file.filename);
const labels = new Set();
// Define path-to-label mappings
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
index efffd0ca3e..cb0bd88057 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main.rs
@@ -77,7 +77,7 @@ fn path_check() -> Job {
"calendar_details_changed",
"${{ steps.filter.outputs.calendar_details_changed }}",
)
- .add_step(checkout("Checkout Repo", false))
+ .add_step(path_check_checkout())
.add_step(paths_filter())
.add_step(detect_schema_changes())
}
@@ -95,37 +95,110 @@ fn typescript() -> Job {
.runs_on(mid_web_runner())
.add_step(checkout("Checkout Repo", false))
.add_step(steps::mount_web_cache_volume(false))
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_reqs_web("Setup Prereqs", false))
- .add_step(check_types())
- .add_step(check_floating_promises())
- .add_step(check_functional_test_types())
- .add_step(check_collaboration_types())
- .add_step(check_lexical_service_types())
- .add_step(test_lexical_service())
.add_step(
Step::new("Install SDK Documentation Dependencies")
.run("bun install --frozen-lockfile")
.working_directory(xtask_paths::repo_dir!("packages/sdk")),
)
- .add_step(
- Step::new("Check SDK Documentation")
- .run("bun run type-check && bun run test")
- .working_directory(xtask_paths::repo_dir!("packages/sdk-docs")),
- )
- .add_step(
- Step::new("Check Editing Worker")
- .run("bun run type-check")
- .working_directory(xtask_paths::repo_dir!("services/ai-editing-worker")),
- )
- .add_step(
- Step::new("Check Code Mode SDK")
- .run("bun run type-check && bun run test")
- .working_directory(xtask_paths::repo_dir!("packages/code-mode-sdk")),
- )
+ .add_step(typecheck_in_parallel())
.add_step(steps::teardown_nix())
}
+/// Typecheck's checks, each `(name, directory, command)`. They read the same
+/// installed tree and write nothing, so they run at once: in sequence they took
+/// about 106 s of a 179 s job. `packages/sdk` isn't a workspace member, so the
+/// SDK documentation's dependencies install first, as they did before.
+const TYPECHECKS: [(&str, &str, &str); 9] = [
+ // `type-check` runs the native TypeScript 7 compiler.
+ ("Check Types", "apps/web", "bun run type-check"),
+ // Type-aware oxlint over all of `apps/web`; `no-floating-promises` is an
+ // error and needs type information to run at all.
+ (
+ "Check Floating Promises",
+ ".",
+ "bun run lint:oxlint --quiet",
+ ),
+ (
+ "Check Functional Test Types",
+ "apps/web",
+ "bun ../../node_modules/typescript-native/bin/tsc --noEmit --skipLibCheck --project tests/functional/tsconfig.json",
+ ),
+ (
+ "Check Collaboration Package Types",
+ "packages/collaboration",
+ "bun run type-check",
+ ),
+ (
+ "Check Lexical Service Types",
+ "services/lexical-service",
+ "bun run check",
+ ),
+ (
+ "Test Lexical Service Endpoints",
+ "services/lexical-service",
+ "bun test src",
+ ),
+ (
+ "Check SDK Documentation",
+ "packages/sdk-docs",
+ "bun run type-check && bun run test",
+ ),
+ (
+ "Check Editing Worker",
+ "services/ai-editing-worker",
+ "bun run type-check",
+ ),
+ (
+ "Check Code Mode SDK",
+ "packages/code-mode-sdk",
+ "bun run type-check && bun run test",
+ ),
+];
+
+/// Runs [`TYPECHECKS`] concurrently. Each check's output is printed as its own
+/// group once all finish, every failure is annotated by name, and the step fails
+/// if any check does.
+fn typecheck_in_parallel() -> Step<Run> {
+ Step::new("Run checks").run(typecheck_script(&TYPECHECKS))
+}
+
+fn typecheck_script(checks: &[(&str, &str, &str)]) -> String {
+ let mut script = String::from(indoc::indoc! {r#"
+ set -uo pipefail
+ logs="$RUNNER_TEMP/typecheck"
+ mkdir -p "$logs"
+ names=()
+ pids=()
+ check() {
+ local name="$1" directory="$2" command="$3"
+ ( cd "$directory" && eval "$command" ) > "$logs/${#names[@]}.log" 2>&1 &
+ names+=("$name")
+ pids+=("$!")
+ }
+ "#});
+ for (name, directory, command) in checks {
+ script.push_str(&format!("check '{name}' '{directory}' '{command}'\n"));
+ }
+ script.push_str(indoc::indoc! {r#"
+ failed=0
+ for index in "${!names[@]}"; do
+ status=0
+ wait "${pids[$index]}" || status=$?
+ echo "::group::${names[$index]} (exit $status)"
+ cat "$logs/$index.log"
+ echo "::endgroup::"
+ if [ "$status" -ne 0 ]; then
+ echo "::error::${names[$index]} failed with exit code $status"
+ failed=1
+ fi
+ done
+ exit "$failed"
+ "#});
+ script
+}
+
/// Everything generated from Rust binaries: the OpenAPI clients in `apps/web`
/// (`gen-api`), the specta wasm wire types, the SDK's specs and generated
/// layer, and the MCP tool reference in `apps/docs`. They used to be checked by
@@ -185,7 +258,7 @@ fn biome_check() -> Job {
gated_web_job("Biome Check")
.add_step(checkout("Checkout Repo", true))
.add_step(steps::mount_nix_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_dev_shell())
.add_step(run_biome())
.add_step(run_collaboration_biome())
@@ -194,16 +267,16 @@ fn biome_check() -> Job {
}
fn test() -> Job {
- gated_web_job("Test (${{ matrix.shard }}/3)")
+ gated_web_job("Test (${{ matrix.shard }}/6)")
.runs_on(mid_web_runner())
.strategy(
Strategy::default()
.fail_fast(false)
- .matrix(serde_json::json!({ "shard": [1, 2, 3] })),
+ .matrix(serde_json::json!({ "shard": [1, 2, 3, 4, 5, 6] })),
)
.add_step(checkout("Checkout Repo", false))
.add_step(steps::mount_web_test_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_reqs_web("Setup", false))
.add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
.add_step(build_agent_fold_for_tests())
@@ -218,7 +291,7 @@ fn functional() -> Job {
.runs_on(mid_web_runner())
.add_step(checkout("Checkout Repo", false))
.add_step(steps::mount_web_build_check_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_reqs_web("Setup", true))
.add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
.add_step(steps::start_sccache_server())
@@ -236,7 +309,7 @@ fn browser() -> Job {
.runs_on(mid_web_runner())
.add_step(checkout("Checkout Repo", false))
.add_step(steps::mount_web_test_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_reqs_web("Setup", true))
.add_step(steps::configure_namespace_sccache(vars::WEB_SCCACHE_NAME))
.add_step(build_agent_fold_for_tests())
@@ -251,13 +324,15 @@ fn cycles() -> Job {
gated_web_job("Cycles Import Check")
.add_step(checkout("Checkout", false))
.add_step(steps::mount_nix_cache_volume())
- .add_step(steps::setup_nix())
+ .add_step(steps::setup_nix_with_seed())
.add_step(steps::setup_dev_shell())
.add_step(cycles_import_check())
.add_step(collaboration_cycles_import_check())
.add_step(steps::teardown_nix())
}
+/// The one web job without the Nix seed: it holds the preview deploy's AWS
+/// credentials, and PR jobs write the seed.
fn build() -> Job {
let (preview_credentials, preview_steps) = preview::steps();
let job = gated_web_job("Build")
@@ -328,6 +403,15 @@ fn checkout(name: &str, full_history: bool) -> Step<Use> {
.when(full_history, |step| step.add_with(("fetch-depth", 0)))
}
+/// The filter reads the PR's files from the API and [`detect_schema_changes`]
+/// reads trees and a few blobs, so path-check fetches no blobs up front and
+/// writes almost no work tree: 18 s became 6 s on the tiny runner.
+fn path_check_checkout() -> Step<Use> {
+ checkout("Checkout Repo", false)
+ .add_with(("filter", "blob:none"))
+ .add_with(("sparse-checkout", ".github"))
+}
+
/// Whether a pull request can change generated output. Schemas come from
/// `utoipa`, `specta` and `schemars` items, so a Rust file naming none of them,
/// before or after the change, can't alter one. Replayed over 1,021 commits on
@@ -336,35 +420,7 @@ fn checkout(name: &str, full_history: bool) -> Step<Use> {
/// that changed generated output. A schema crate's version change still counts.
fn detect_schema_changes() -> Step<Run> {
Step::new("Detect schema changes")
- .run(indoc::indoc! {r#"
- set -euo pipefail
- result() {
- echo "$1"
- echo "codegen_changed=$2" >> "$GITHUB_OUTPUT"
- exit 0
- }
- [ "$CODEGEN_INPUTS" = true ] && result "A generator input changed." true
- # The merge commit's first parent is the main this merge result was
- # built on; a fresh fetch of main could be newer.
- base=$(git cat-file -p HEAD | sed -n 's/^parent //p' | sed -n 1p)
- [ -n "$base" ] || result "No base commit to compare with." true
- git fetch -q --depth=1 origin "$base"
- names='utoipa|ToSchema|IntoParams|specta|schemars|JsonSchema'
- # Every grep here reads all its input. One that stopped at its first
- # match would close the pipe, and pipefail reads the writer's SIGPIPE as
- # no match.
- if git diff "$base" HEAD -- Cargo.toml Cargo.lock | grep -E "^[-+].*($names)"; then
- result "A schema crate's version changed." true
- fi
- while IFS= read -r file; do
- for revision in HEAD "$base"; do
- if git show "$revision:$file" 2>/dev/null | grep -E "$names" > /dev/null; then
- result "$file names a schema item." true
- fi
- done
- done < <(git diff --name-only --no-renames "$base" HEAD -- 'crates/*.rs' 'services/*.rs')
- result "No changed Rust file names a schema item." false
- "#})
+ .run(DETECT_SCHEMA_CHANGES)
.id("codegen")
.add_env((
"CODEGEN_INPUTS",
@@ -372,6 +428,42 @@ fn detect_schema_changes() -> Step<Run> {
))
}
+/// The checkout is blobless, so `git diff` and `git show` fetch blobs as they
+/// read them. Each read is its own command, so a failed fetch fails the step
+/// instead of reading as "no schema item"; a file absent from one revision is
+/// skipped by a tree lookup, which needs no blob.
+const DETECT_SCHEMA_CHANGES: &str = indoc::indoc! {r#"
+ set -euo pipefail
+ result() {
+ echo "$1"
+ echo "codegen_changed=$2" >> "$GITHUB_OUTPUT"
+ exit 0
+ }
+ [ "$CODEGEN_INPUTS" = true ] && result "A generator input changed." true
+ # The merge commit's first parent is the main this merge result was
+ # built on; a fresh fetch of main could be newer.
+ base=$(git cat-file -p HEAD | sed -n 's/^parent //p' | sed -n 1p)
+ [ -n "$base" ] || result "No base commit to compare with." true
+ git fetch -q --depth=1 origin "$base"
+ names='utoipa|ToSchema|IntoParams|specta|schemars|JsonSchema'
+ manifests=$(git diff "$base" HEAD -- Cargo.toml Cargo.lock)
+ if grep -E "^[-+].*($names)" <<< "$manifests"; then
+ result "A schema crate's version changed." true
+ fi
+ changed=$(git diff --name-only --no-renames "$base" HEAD -- 'crates/*.rs' 'services/*.rs')
+ while IFS= read -r file; do
+ [ -n "$file" ] || continue
+ for revision in HEAD "$base"; do
+ [ -n "$(git ls-tree "$revision" -- "$file")" ] || continue
+ contents=$(git show "$revision:$file")
+ if grep -E "$names" <<< "$contents" > /dev/null; then
+ result "$file names a schema item." true
+ fi
+ done
+ done <<< "$changed"
+ result "No changed Rust file names a schema item." false
+"#};
+
fn paths_filter() -> Step<Use> {
let artifact_paths = crate::workflows::web_artifact_paths::yaml_list(" ");
@@ -480,43 +572,6 @@ fi
))
}
-/// `type-check` runs the native TypeScript 7 compiler.
-fn check_types() -> Step<Run> {
- Step::new("Check Types")
- .run("bun run type-check")
- .working_directory(xtask_paths::repo_dir!("apps/web"))
-}
-
-/// Type-aware oxlint over all of `apps/web`; `no-floating-promises` is an
-/// error and needs type information to run at all.
-fn check_floating_promises() -> Step<Run> {
- Step::new("Check Floating Promises").run("bun run lint:oxlint --quiet")
-}
-
-fn check_functional_test_types() -> Step<Run> {
- Step::new("Check Functional Test Types")
- .run("bun ../../node_modules/typescript-native/bin/tsc --noEmit --skipLibCheck --project tests/functional/tsconfig.json")
- .working_directory(xtask_paths::repo_dir!("apps/web"))
-}
-
-fn check_collaboration_types() -> Step<Run> {
- Step::new("Check Collaboration Package Types")
- .run("bun run type-check")
- .working_directory(xtask_paths::repo_dir!("packages/collaboration"))
-}
-
-fn check_lexical_service_types() -> Step<Run> {
- Step::new("Check Lexical Service Types")
- .run("bun run check")
- .working_directory(xtask_paths::repo_dir!("services/lexical-service"))
-}
-
-fn test_lexical_service() -> Step<Run> {
- Step::new("Test Lexical Service Endpoints")
- .run("bun test src")
- .working_directory(xtask_paths::repo_dir!("services/lexical-service"))
-}
-
fn run_biome() -> Step<Run> {
Step::new("Run Biome")
.run("biome ci --changed --no-errors-on-unmatched --error-on-warnings")
@@ -550,7 +605,7 @@ fn build_agent_fold_for_tests() -> Step<Run> {
/// keeps it warm across jobs.
fn run_tests() -> Step<Run> {
Step::new("Test")
- .run("bun run test --shard=${{ matrix.shard }}/3")
+ .run("bun run test --shard=${{ matrix.shard }}/6")
.add_env((
"VITEST_MODULE_CACHE_DIR",
vars::VITEST_MODULE_CACHE_VOLUME_DIR,
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
index a28861d497..1ed13dd330 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/web_app_check_main/test.rs
@@ -170,15 +170,26 @@ fn sdk_and_documentation_changes_run_their_checks() {
assert!(should_run.contains("packages/sdk/**"));
assert!(should_run.contains("packages/sdk-docs/**"));
let typescript = job(&yaml, "typescript", "generated-code");
- assert!(typescript.contains("name: Install SDK Documentation Dependencies"));
- assert!(typescript.contains("name: Check SDK Documentation"));
- assert!(typescript.contains("working-directory: packages/sdk-docs"));
+ let install = typescript
+ .find("name: Install SDK Documentation Dependencies")
+ .expect("SDK documentation dependencies install");
+ let checks = typescript
+ .find("- name: Run checks\n")
+ .expect("parallel checks step");
+ assert!(
+ install < checks,
+ "the SDK documentation's dependencies install first"
+ );
+ assert!(typescript.contains("check 'Check SDK Documentation' 'packages/sdk-docs'"));
+ assert!(typescript.contains("check 'Check Code Mode SDK' 'packages/code-mode-sdk'"));
}
#[test]
fn typecheck_compiles_no_rust() {
let yaml = web_app_check_main().to_string().expect("workflow yaml");
- let typescript = job(&yaml, "typescript", "generated-code");
+ // The Nix seed's cache shares sccache's name; that's not a compile.
+ let typescript =
+ job(&yaml, "typescript", "generated-code").replace("seed-cache: sccache-ci", "");
for rust in ["gen-api", "cargo", "sccache", "just gen-"] {
assert!(
!typescript.contains(rust),
@@ -372,7 +383,7 @@ fn build_job_uses_remote_sccache_and_wasm_cache() {
/// The Test legs, up to the next job.
fn test_job(yaml: &str) -> &str {
let start = yaml
- .find("name: Test (${{ matrix.shard }}/3)")
+ .find("name: Test (${{ matrix.shard }}/6)")
.expect("Test job");
let end = yaml[start..]
.find("name: Functional Tests\n")
@@ -408,7 +419,7 @@ fn test_job_shards_vitest_after_building_its_wasm() {
.find("just build-agent-fold-wasm")
.expect("build WASM before the test setup hook");
let vitest = test_job
- .find("bun run test --shard=${{ matrix.shard }}/3")
+ .find("bun run test --shard=${{ matrix.shard }}/6")
.expect("vitest step runs one shard");
assert!(wasm < vitest, "cold WASM compilation precedes Vitest");
assert!(
@@ -780,3 +791,301 @@ fn status_check_rejects_a_cancelled_generated_code_job() {
let yaml = web_app_check_main().to_string().expect("workflow yaml");
assert!(yaml.contains(r#"[[ "${{ needs.generated-code.result }}" == "cancelled" ]]"#));
}
+
+/// The checks run at once, and the step still fails, naming the check, when any
+/// one of them fails; every check's output is printed either way.
+#[test]
+fn typecheck_runs_its_checks_together_and_fails_on_any() {
+ let run = |checks: &[(&str, &str, &str)]| {
+ let root = tempfile::tempdir().expect("temporary checkout");
+ for (_, directory, _) in checks {
+ std::fs::create_dir_all(root.path().join(directory)).expect("check directory");
+ }
+ let output = std::process::Command::new("bash")
+ .arg("-eo")
+ .arg("pipefail")
+ .arg("-c")
+ .arg(typecheck_script(checks))
+ .current_dir(root.path())
+ .env("RUNNER_TEMP", root.path())
+ .output()
+ .expect("run the Typecheck script");
+ (
+ output.status.success(),
+ String::from_utf8_lossy(&output.stdout).into_owned(),
+ )
+ };
+
+ let (passed, log) = run(&[
+ ("Check Types", "apps/web", "echo types ok"),
+ ("Check Editing Worker", "services/worker", "echo worker ok"),
+ ]);
+ assert!(passed, "{log}");
+ assert!(
+ log.contains("::group::Check Types (exit 0)\ntypes ok\n::endgroup::"),
+ "{log}"
+ );
+ assert!(log.contains("worker ok"), "{log}");
+
+ let (passed, log) = run(&[
+ ("Check Types", "apps/web", "sleep 1; echo types ok"),
+ ("Check Floating Promises", ".", "echo lint failed; exit 3"),
+ ("Check Editing Worker", "services/worker", "echo worker ok"),
+ ]);
+ assert!(!passed, "{log}");
+ assert!(
+ log.contains("::error::Check Floating Promises failed with exit code 3"),
+ "{log}"
+ );
+ assert!(
+ log.contains("types ok") && log.contains("worker ok"),
+ "{log}"
+ );
+}
+
+#[test]
+fn every_former_typecheck_step_still_runs() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let typescript = job(&yaml, "typescript", "generated-code");
+ for (name, directory, command) in [
+ ("Check Types", "apps/web", "bun run type-check"),
+ (
+ "Check Floating Promises",
+ ".",
+ "bun run lint:oxlint --quiet",
+ ),
+ (
+ "Check Functional Test Types",
+ "apps/web",
+ "--project tests/functional/tsconfig.json",
+ ),
+ (
+ "Check Collaboration Package Types",
+ "packages/collaboration",
+ "bun run type-check",
+ ),
+ (
+ "Check Lexical Service Types",
+ "services/lexical-service",
+ "bun run check",
+ ),
+ (
+ "Test Lexical Service Endpoints",
+ "services/lexical-service",
+ "bun test src",
+ ),
+ (
+ "Check SDK Documentation",
+ "packages/sdk-docs",
+ "bun run type-check && bun run test",
+ ),
+ (
+ "Check Editing Worker",
+ "services/ai-editing-worker",
+ "bun run type-check",
+ ),
+ (
+ "Check Code Mode SDK",
+ "packages/code-mode-sdk",
+ "bun run type-check && bun run test",
+ ),
+ ] {
+ let line = typescript
+ .lines()
+ .find(|line| line.contains(&format!("check '{name}' '{directory}' ")))
+ .unwrap_or_else(|| panic!("{name} must run in {directory}: {typescript}"));
+ assert!(
+ line.contains(command),
+ "{name} must run `{command}`: {line}"
+ );
+ }
+}
+
+#[test]
+fn path_check_fetches_no_blobs_up_front() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ let path_check = job(&yaml, "path-check", "android-native-check");
+ assert!(path_check.contains("filter: blob:none"), "{path_check}");
+ assert!(
+ path_check.contains("sparse-checkout: '.github'"),
+ "{path_check}"
+ );
+}
+
+/// PR jobs write the seed, so the job holding the preview's AWS credentials
+/// doesn't read it.
+#[test]
+fn web_jobs_restore_the_nix_seed_except_the_build() {
+ let yaml = web_app_check_main().to_string().expect("workflow yaml");
+ for (id, next) in [
+ ("typescript", "generated-code"),
+ ("generated-code", "biome-check"),
+ ("biome-check", "test"),
+ ("test", "functional"),
+ ("functional", "browser"),
+ ("browser", "cycles"),
+ ("cycles", "build"),
+ ] {
+ let job = job(&yaml, id, next);
+ assert!(
+ job.contains("seed-cache: sccache-ci"),
+ "{id} restores the Nix seed: {job}"
+ );
+ }
+ let build = build_job(&yaml);
+ assert!(!build.contains("seed-cache"), "{build}");
+ assert!(
+ build.contains("aws-access-key-id"),
+ "Build holds the preview credentials"
+ );
+}
+
+/// Runs the schema detection step in a blobless clone, as path-check checks
+/// out. `files` are written in a commit on top of a base commit. With
+/// `break_blobs`, origin loses those files' blobs, so fetching them fails.
+fn detect_schema_changes_in_blobless_clone(
+ files: &[(&str, &str)],
+ break_blobs: bool,
+) -> (bool, String) {
+ use std::process::Command;
+
+ let root = tempfile::tempdir().expect("temporary directory");
+ let git = |dir: &std::path::Path, args: &[&str]| {
+ let output = Command::new("git")
+ .args(args)
+ .current_dir(dir)
+ .env("GIT_CONFIG_GLOBAL", "/dev/null")
+ .env("GIT_CONFIG_NOSYSTEM", "1")
+ .env("GIT_AUTHOR_NAME", "test")
+ .env("GIT_AUTHOR_EMAIL", "[email protected]")
+ .env("GIT_COMMITTER_NAME", "test")
+ .env("GIT_COMMITTER_EMAIL", "[email protected]")
+ .output()
+ .expect("run git");
+ assert!(
+ output.status.success(),
+ "git {args:?}: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ String::from_utf8_lossy(&output.stdout).trim().to_string()
+ };
+
+ let source = root.path().join("source");
+ std::fs::create_dir_all(source.join("crates/server/src")).expect("source tree");
+ git(&source, &["init", "-q"]);
+ std::fs::write(source.join("Cargo.toml"), "[workspace]\n").expect("manifest");
+ std::fs::write(
+ source.join("crates/server/src/lib.rs"),
+ "pub fn serve() {}\n",
+ )
+ .expect("lib");
+ git(&source, &["add", "-A"]);
+ git(&source, &["commit", "-q", "-m", "base"]);
+ for (path, contents) in files {
+ let path = source.join(path);
+ std::fs::create_dir_all(path.parent().expect("parent")).expect("file directory");
+ std::fs::write(path, contents).expect("changed file");
+ }
+ git(&source, &["add", "-A"]);
+ git(&source, &["commit", "-q", "-m", "change"]);
+
+ let origin = root.path().join("origin.git");
+ git(
+ root.path(),
+ &[
+ "clone",
+ "-q",
+ "--bare",
+ source.to_str().expect("path"),
+ origin.to_str().expect("path"),
+ ],
+ );
+ git(&origin, &["config", "uploadpack.allowFilter", "true"]);
+ git(
+ &origin,
+ &["config", "uploadpack.allowAnySHA1InWant", "true"],
+ );
+ let url = format!("file://{}", origin.display());
+ let checkout = root.path().join("checkout");
+ git(
+ root.path(),
+ &[
+ "clone",
+ "-q",
+ "--filter=blob:none",
+ "--no-checkout",
+ "--depth=1",
+ &url,
+ checkout.to_str().expect("path"),
+ ],
+ );
+ if break_blobs {
+ for (path, _) in files {
+ let blob = git(&origin, &["rev-parse", &format!("HEAD:{path}")]);
+ std::fs::remove_file(origin.join("objects").join(&blob[..2]).join(&blob[2..]))
+ .expect("remove the blob from origin");
+ }
+ }
+
+ let output_file = root.path().join("github_output");
+ let output = Command::new("bash")
+ .arg("-c")
+ .arg(DETECT_SCHEMA_CHANGES)
+ .current_dir(&checkout)
+ .env("CODEGEN_INPUTS", "false")
+ .env("GITHUB_OUTPUT", &output_file)
+ .env("GIT_CONFIG_GLOBAL", "/dev/null")
+ .env("GIT_CONFIG_NOSYSTEM", "1")
+ .output()
+ .expect("run schema detection");
+ let outputs = std::fs::read_to_string(&output_file).unwrap_or_default();
+ (
+ output.status.success(),
+ format!(
+ "{outputs}{}{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ ),
+ )
+}
+
+#[test]
+fn schema_detection_reads_blobs_on_demand_and_fails_when_it_cannot() {
+ let schema_file = [(
+ "crates/server/src/api.rs",
+ "#[derive(utoipa::ToSchema)]\npub struct Reply;\n",
+ )];
+
+ let (passed, log) = detect_schema_changes_in_blobless_clone(&schema_file, false);
+ assert!(
+ passed && log.contains("codegen_changed=true"),
+ "a new file naming a schema item: {log}"
+ );
+
+ let (passed, log) = detect_schema_changes_in_blobless_clone(
+ &[("crates/server/src/lib.rs", "pub fn serve() { listen() }\n")],
+ false,
+ );
+ assert!(
+ passed && log.contains("codegen_changed=false"),
+ "no schema item: {log}"
+ );
+
+ let (passed, log) = detect_schema_changes_in_blobless_clone(
+ &[(
+ "Cargo.toml",
+ "[workspace]\n[workspace.dependencies]\nutoipa = \"5\"\n",
+ )],
+ false,
+ );
+ assert!(
+ passed && log.contains("codegen_changed=true"),
+ "a schema crate's version: {log}"
+ );
+
+ let (passed, log) = detect_schema_changes_in_blobless_clone(&schema_file, true);
+ assert!(
+ !passed && !log.contains("codegen_changed="),
+ "a failed blob fetch must fail the step: {log}"
+ );
+}