Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

perf(config): scope CI from canonical plan (#700)

Genera3541b221b2c
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f7493dfaf..920637b73 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,9 +8,33 @@ on:
merge_group:
types: [checks_requested]
workflow_dispatch:
+ inputs:
+ mode:
+ description: Run the full contract or a guarded diagnostic characterization fixture
+ type: choice
+ options: [full, characterization]
+ default: full
+ candidate_pr:
+ description: Candidate PR number (characterization only)
+ type: string
+ required: false
+ candidate_sha:
+ description: Exact candidate PR head SHA (characterization only)
+ type: string
+ required: false
+ fixture_sha:
+ description: Exact single-change fixture SHA (characterization only)
+ type: string
+ required: false
+ expected_class:
+ description: Expected non-full classifier class (blog, package, or shared)
+ type: string
+ required: false
permissions:
+ actions: read
contents: read
+ pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_sha || github.ref }}
@@ -22,32 +46,87 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
+ execution_plan: ${{ steps.scope.outputs.execution_plan }}
+ plan_hash: ${{ steps.scope.outputs.plan_hash }}
+ plan_schema_version: ${{ steps.scope.outputs.plan_schema_version }}
full: ${{ steps.scope.outputs.full }}
base_sha: ${{ steps.scope.outputs.base_sha }}
head_sha: ${{ steps.scope.outputs.head_sha }}
blog_only: ${{ steps.scope.outputs.blog_only }}
run_blog_build: ${{ steps.scope.outputs.run_blog_build }}
run_code_checks: ${{ steps.scope.outputs.run_code_checks }}
+ run_static_checks: ${{ steps.scope.outputs.run_static_checks }}
+ run_linux: ${{ steps.scope.outputs.run_linux }}
run_windows: ${{ steps.scope.outputs.run_windows }}
run_e2e: ${{ steps.scope.outputs.run_e2e }}
+ run_ui_e2e: ${{ steps.scope.outputs.run_ui_e2e }}
run_rust: ${{ steps.scope.outputs.run_rust }}
run_lighthouse: ${{ steps.scope.outputs.run_lighthouse }}
run_knip: ${{ steps.scope.outputs.run_knip }}
run_app_packed_artifact: ${{ steps.scope.outputs.run_app_packed_artifact }}
+ static_packages: ${{ steps.scope.outputs.static_packages }}
+ build_packages: ${{ steps.scope.outputs.build_packages }}
+ knip_packages: ${{ steps.scope.outputs.knip_packages }}
+ unit_test_roots: ${{ steps.scope.outputs.unit_test_roots }}
+ windows_suites: ${{ steps.scope.outputs.windows_suites }}
+ integration_suites: ${{ steps.scope.outputs.integration_suites }}
+ linux_suites: ${{ steps.scope.outputs.linux_suites }}
+ coverage_targets: ${{ steps.scope.outputs.coverage_targets }}
+ coverage_include_roots: ${{ steps.scope.outputs.coverage_include_roots }}
+ coverage_required_changed_files: ${{ steps.scope.outputs.coverage_required_changed_files }}
+ ui_specs: ${{ steps.scope.outputs.ui_specs }}
+ e2e_matrix: ${{ steps.scope.outputs.e2e_matrix }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- id: scope
env:
- BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.event.before }}
- HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
- FORCE_FULL: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ EVENT_NAME: ${{ github.event_name }}
+ EVENT_SHA: ${{ github.sha }}
+ PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ MERGE_BASE_SHA: ${{ github.event.merge_group.base_sha }}
+ PUSH_BASE_SHA: ${{ github.event.before }}
+ DISPATCH_MODE: ${{ inputs.mode }}
+ CANDIDATE_PR: ${{ inputs.candidate_pr }}
+ CANDIDATE_SHA: ${{ inputs.candidate_sha }}
+ FIXTURE_SHA: ${{ inputs.fixture_sha }}
+ EXPECTED_CLASS: ${{ inputs.expected_class }}
+ GH_TOKEN: ${{ github.token }}
run: |
- printf 'base_sha=%s\nhead_sha=%s\n' "$BASE_SHA" "$HEAD_SHA" >> "$GITHUB_OUTPUT"
- args=(--base "$BASE_SHA" --head "$HEAD_SHA" --output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY")
- if [[ "$FORCE_FULL" == "true" ]]; then args+=(--force-full); fi
+ set -euo pipefail
+ args=(--output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" --plan-file .ci/execution-plan.json)
+ mkdir -p .ci
+ if [[ "$EVENT_NAME" == "workflow_dispatch" && "$DISPATCH_MODE" == "characterization" ]]; then
+ [[ "$CANDIDATE_PR" =~ ^[1-9][0-9]*$ ]]
+ gh api "repos/$GITHUB_REPOSITORY/pulls/$CANDIDATE_PR" > "$RUNNER_TEMP/candidate-pr.json"
+ node scripts/ci/characterization-guard.mjs \
+ --repository "$GITHUB_REPOSITORY" \
+ --ref-name "$GITHUB_REF_NAME" \
+ --event-sha "$EVENT_SHA" \
+ --candidate-sha "$CANDIDATE_SHA" \
+ --fixture-sha "$FIXTURE_SHA" \
+ --expected-class "$EXPECTED_CLASS" \
+ --candidate-pr-json "$RUNNER_TEMP/candidate-pr.json" \
+ --summary "$GITHUB_STEP_SUMMARY"
+ args+=(--base "$CANDIDATE_SHA" --head "$FIXTURE_SHA" --diagnostic-only)
+ elif [[ "$EVENT_NAME" == "pull_request" ]]; then
+ args+=(--base "$PR_BASE_SHA" --head "$PR_HEAD_SHA")
+ elif [[ "$EVENT_NAME" == "merge_group" ]]; then
+ args+=(--base "$MERGE_BASE_SHA" --head "$EVENT_SHA")
+ elif [[ "$EVENT_NAME" == "push" ]]; then
+ args+=(--base "$PUSH_BASE_SHA" --head "$EVENT_SHA" --force-full)
+ else
+ args+=(--base "$EVENT_SHA" --head "$EVENT_SHA" --force-full)
+ fi
node scripts/ci/changed-scopes.mjs "${args[@]}"
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: execution-plan-${{ github.run_id }}-${{ github.run_attempt }}
+ path: .ci/execution-plan.json
+ if-no-files-found: error
+ retention-days: 14
blog-build:
name: Blog build
@@ -57,6 +136,13 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: run_blog_build
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.run_blog_build }}
+ run: node scripts/ci/validate-execution-plan.mjs
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
@@ -79,13 +165,11 @@ jobs:
static-checks:
name: Static checks
needs: scope
- if: needs.scope.outputs.run_code_checks == 'true'
+ if: needs.scope.outputs.run_static_checks == 'true'
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- fetch-depth: 0
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
@@ -101,55 +185,57 @@ jobs:
restore-keys: |
turbo-${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-
- run: pnpm install --frozen-lockfile
- - if: needs.scope.outputs.full == 'true'
- run: pnpm typecheck
- - if: needs.scope.outputs.full != 'true'
+ - name: Validate execution plan
env:
- TURBO_SCM_BASE: ${{ needs.scope.outputs.base_sha }}
- TURBO_SCM_HEAD: ${{ needs.scope.outputs.head_sha }}
- run: pnpm turbo run typecheck --affected
- - if: needs.scope.outputs.full == 'true'
- run: pnpm lint
- - if: needs.scope.outputs.full != 'true'
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: static_packages
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.static_packages }}
+ run: node scripts/ci/validate-execution-plan.mjs
+ - name: Typecheck selected packages
env:
- TURBO_SCM_BASE: ${{ needs.scope.outputs.base_sha }}
- TURBO_SCM_HEAD: ${{ needs.scope.outputs.head_sha }}
- run: pnpm turbo run lint --affected
+ SELECTED_PACKAGES: ${{ needs.scope.outputs.static_packages }}
+ run: |
+ mapfile -t packages < <(node -e 'for (const value of JSON.parse(process.env.SELECTED_PACKAGES)) console.log(value)')
+ filters=(); for package in "${packages[@]}"; do filters+=("--filter=$package"); done
+ pnpm turbo run typecheck "${filters[@]}"
+ - name: Lint selected packages
+ env:
+ SELECTED_PACKAGES: ${{ needs.scope.outputs.static_packages }}
+ run: |
+ mapfile -t packages < <(node -e 'for (const value of JSON.parse(process.env.SELECTED_PACKAGES)) console.log(value)')
+ filters=(); for package in "${packages[@]}"; do filters+=("--filter=$package"); done
+ pnpm turbo run lint "${filters[@]}"
- run: pnpm typegrep:ws
- run: pnpm typegrep:agent-driver
- - name: Build
- if: needs.scope.outputs.full == 'true'
- run: pnpm build --filter=@alook/shared --filter=@alook/web --filter=@alook/cli --filter=@alook/email-worker --filter=@alook/ws-do --filter=@alook/wake-worker
- - name: Build affected packages
- if: needs.scope.outputs.full != 'true'
+ - name: Build selected packages
+ if: needs.scope.outputs.build_packages != '[]'
env:
- TURBO_SCM_BASE: ${{ needs.scope.outputs.base_sha }}
- TURBO_SCM_HEAD: ${{ needs.scope.outputs.head_sha }}
- run: pnpm turbo run build --affected --filter=@alook/shared --filter=@alook/web --filter=@alook/cli --filter=@alook/email-worker --filter=@alook/ws-do --filter=@alook/wake-worker
- - name: Dead code detection
- if: needs.scope.outputs.run_knip == 'true' && needs.scope.outputs.full == 'true'
- continue-on-error: true
- run: pnpm knip
- - name: Dead code detection for affected packages
- if: needs.scope.outputs.run_knip == 'true' && needs.scope.outputs.full != 'true'
+ SELECTED_PACKAGES: ${{ needs.scope.outputs.build_packages }}
+ run: |
+ mapfile -t packages < <(node -e 'for (const value of JSON.parse(process.env.SELECTED_PACKAGES)) console.log(value)')
+ filters=(); for package in "${packages[@]}"; do filters+=("--filter=$package"); done
+ pnpm turbo run build "${filters[@]}"
+ - name: Dead code detection for selected packages
+ if: needs.scope.outputs.knip_packages != '[]'
continue-on-error: true
env:
- TURBO_SCM_BASE: ${{ needs.scope.outputs.base_sha }}
- TURBO_SCM_HEAD: ${{ needs.scope.outputs.head_sha }}
- run: pnpm turbo run knip --affected
+ SELECTED_PACKAGES: ${{ needs.scope.outputs.knip_packages }}
+ run: |
+ mapfile -t packages < <(node -e 'for (const value of JSON.parse(process.env.SELECTED_PACKAGES)) console.log(value)')
+ filters=(); for package in "${packages[@]}"; do filters+=("--filter=$package"); done
+ pnpm turbo run knip "${filters[@]}"
test-linux:
name: Tests & Coverage (ubuntu-latest)
needs: scope
- if: needs.scope.outputs.run_code_checks == 'true'
+ if: needs.scope.outputs.run_linux == 'true'
runs-on: ubuntu-latest
timeout-minutes: 25
env:
RUN_COVERAGE: ${{ github.event_name != 'push' || startsWith(github.event.head_commit.message, 'release:') }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- fetch-depth: 0
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
@@ -165,53 +251,120 @@ jobs:
restore-keys: |
turbo-${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-
- run: pnpm install --frozen-lockfile
- - name: Build agent-driver for coverage
- if: env.RUN_COVERAGE == 'true'
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: unit_test_roots
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.unit_test_roots }}
+ run: node scripts/ci/validate-execution-plan.mjs
+ - name: Build selected agent-driver fixture
+ if: >-
+ contains(fromJSON(needs.scope.outputs.unit_test_roots), 'src/daemon/agent-driver') ||
+ contains(fromJSON(needs.scope.outputs.unit_test_roots), 'src/daemon') ||
+ contains(fromJSON(needs.scope.outputs.linux_suites), 'agent-driver-packed') ||
+ contains(fromJSON(needs.scope.outputs.linux_suites), 'daemon-packed') ||
+ contains(fromJSON(needs.scope.outputs.linux_suites), 'daemon-real')
run: pnpm --filter @alook/agent-driver build
- - name: Run non-daemon tests with coverage
- if: env.RUN_COVERAGE == 'true'
- run: |
- pnpm vitest run --project='!daemon-node' --coverage \
- --reporter=blob --outputFile=.vitest-reports/non-daemon.blob \
- --exclude src/pack.test.ts
- - name: Run daemon tests with coverage
- if: env.RUN_COVERAGE == 'true'
+ - name: Run selected unit tests
+ env:
+ UNIT_ROOTS: ${{ needs.scope.outputs.unit_test_roots }}
+ COVERAGE_ROOTS: ${{ needs.scope.outputs.coverage_include_roots }}
run: |
- pnpm vitest run --project=daemon-node --coverage \
- --reporter=blob --outputFile=.vitest-reports/daemon.blob \
- --exclude src/cli/daemonLifecycle.real.test.ts \
- --exclude src/cli/diagnosticsLifecycle.real.test.ts \
- --exclude src/cli/daemonStop.test.ts \
- --exclude src/version.packed.test.ts \
- --exclude src/agent-driver-bundle.packed.test.ts \
- --exclude src/cli/daemonSelfUpdate.real.test.ts
- - name: Merge coverage reports
+ set -euo pipefail
+ mapfile -t roots < <(node -e 'for (const value of JSON.parse(process.env.UNIT_ROOTS)) console.log(value)')
+ mapfile -t coverage_roots < <(node -e 'for (const value of JSON.parse(process.env.COVERAGE_ROOTS)) console.log(value)')
+ daemon=false; non_daemon=(); daemon_coverage=()
+ for root in "${roots[@]}"; do
+ if [[ "$root" == "src/daemon" ]]; then daemon=true; else non_daemon+=("$root"); fi
+ done
+ for root in "${coverage_roots[@]}"; do
+ if [[ "$root" == src/daemon/src/* || "$root" == src/daemon/scripts/* ]]; then
+ daemon_coverage+=("--coverage.include=$root")
+ fi
+ done
+ report_dir="$PWD/.vitest-reports"
+ coverage_dir="$RUNNER_TEMP/vitest-coverage"
+ mkdir -p "$report_dir"
+ mkdir -p "$coverage_dir"
+ root_index=0
+ for root in "${non_daemon[@]}"; do
+ ((root_index += 1))
+ root_coverage=()
+ report_stem=${root//\//-}
+ if [[ "$RUN_COVERAGE" == "true" ]]; then
+ for include in "${coverage_roots[@]}"; do
+ if [[ "$include" == "$root/"* ]]; then
+ root_coverage+=("--coverage.include=$include")
+ fi
+ done
+ if (( ${#root_coverage[@]} )); then
+ root_coverage=(
+ --coverage
+ --coverage.reporter=json
+ "--coverage.reportsDirectory=$coverage_dir/root-${root_index}-${report_stem}"
+ "${root_coverage[@]}"
+ )
+ fi
+ fi
+ case "$root" in
+ scripts/ci) projects=(ci-scripts) ;;
+ src/email-worker) projects=(email-worker-node email-worker-runtime) ;;
+ src/wake-worker) projects=(wake-worker-node wake-worker-runtime) ;;
+ src/web) projects=(web-node web-runtime) ;;
+ src/ws-do) projects=(ws-do-node ws-do-runtime) ;;
+ *) projects=("$(node -e 'console.log(require("./" + process.argv[1] + "/package.json").name)' "$root")") ;;
+ esac
+ project_args=()
+ for project in "${projects[@]}"; do project_args+=("--project=$project"); done
+ vitest_args=("${project_args[@]}")
+ if (( ${#root_coverage[@]} )); then vitest_args+=("${root_coverage[@]}"); fi
+ pnpm vitest run "${vitest_args[@]}" \
+ --reporter=default --reporter=blob \
+ --outputFile.blob="$report_dir/root-${root_index}-${report_stem}.blob" \
+ --exclude src/pack.test.ts
+ done
+ if [[ "$daemon" == "true" ]]; then
+ coverage=(); if [[ "$RUN_COVERAGE" == "true" ]]; then coverage=(--coverage); fi
+ VITEST_MAX_WORKERS=1 pnpm vitest run --project=daemon-node src/daemon "${coverage[@]}" \
+ "${daemon_coverage[@]}" --reporter=default --reporter=blob \
+ --outputFile.blob="$report_dir/daemon.blob" \
+ --exclude src/cli/daemonLifecycle.real.test.ts \
+ --exclude src/cli/diagnosticsLifecycle.real.test.ts \
+ --exclude src/cli/daemonStop.test.ts \
+ --exclude src/version.packed.test.ts \
+ --exclude src/agent-driver-bundle.packed.test.ts \
+ --exclude src/cli/daemonSelfUpdate.real.test.ts
+ fi
+ - name: Merge and verify selected coverage
if: env.RUN_COVERAGE == 'true'
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
run: |
+ printf '%s\n' "$CI_EXECUTION_PLAN" > "$RUNNER_TEMP/execution-plan.json"
pnpm vitest run --coverage --coverage.reporter=json --coverage.reporter=text \
--merge-reports=.vitest-reports
- - name: Run daemon real-process tests after coverage
- if: env.RUN_COVERAGE == 'true'
+ node scripts/ci/verify-coverage-plan.mjs \
+ --plan "$RUNNER_TEMP/execution-plan.json" \
+ --report coverage/coverage-final.json \
+ --output "$RUNNER_TEMP/coverage-target-manifest.json"
+ - name: Run selected daemon real-process tests
+ if: contains(fromJSON(needs.scope.outputs.linux_suites), 'daemon-real')
run: |
pnpm --filter @alook/daemon exec vitest run --no-file-parallelism \
src/cli/daemonLifecycle.real.test.ts src/cli/diagnosticsLifecycle.real.test.ts \
src/cli/daemonStop.test.ts
- - name: Run package artifact tests after coverage
- if: env.RUN_COVERAGE == 'true'
- run: |
- pnpm --filter @alook/agent-driver exec vitest run --no-file-parallelism src/pack.test.ts
- pnpm --filter @alook/daemon exec vitest run --no-file-parallelism src/version.packed.test.ts src/agent-driver-bundle.packed.test.ts src/cli/daemonSelfUpdate.real.test.ts
- - name: Run daemon tests in isolation
- if: env.RUN_COVERAGE != 'true'
+ - name: Run selected package artifact tests
+ if: contains(fromJSON(needs.scope.outputs.linux_suites), 'agent-driver-packed') || contains(fromJSON(needs.scope.outputs.linux_suites), 'daemon-packed')
env:
- VITEST_MAX_WORKERS: 1
- run: pnpm turbo run test --filter=@alook/daemon
- - name: Run remaining workspace tests
- if: env.RUN_COVERAGE != 'true'
- run: pnpm turbo run test --filter='!@alook/daemon'
- - name: Run CI script tests
- if: env.RUN_COVERAGE != 'true'
- run: pnpm vitest run --project ci-scripts
+ LINUX_SUITES: ${{ needs.scope.outputs.linux_suites }}
+ run: |
+ if node -e 'process.exit(JSON.parse(process.env.LINUX_SUITES).includes("agent-driver-packed") ? 0 : 1)'; then
+ pnpm --filter @alook/agent-driver exec vitest run --no-file-parallelism src/pack.test.ts
+ fi
+ if node -e 'process.exit(JSON.parse(process.env.LINUX_SUITES).includes("daemon-packed") ? 0 : 1)'; then
+ pnpm --filter @alook/daemon exec vitest run --no-file-parallelism src/version.packed.test.ts src/agent-driver-bundle.packed.test.ts src/cli/daemonSelfUpdate.real.test.ts
+ fi
- name: Upload coverage
if: env.RUN_COVERAGE == 'true'
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
@@ -219,6 +372,17 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage/coverage-final.json
fail_ci_if_error: true
+ - name: Upload coverage plan evidence
+ if: env.RUN_COVERAGE == 'true'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: coverage-plan-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ ${{ runner.temp }}/execution-plan.json
+ ${{ runner.temp }}/coverage-target-manifest.json
+ coverage/coverage-final.json
+ if-no-files-found: error
+ retention-days: 14
test-windows:
name: Tests (windows-latest)
@@ -228,8 +392,6 @@ jobs:
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- fetch-depth: 0
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
@@ -245,11 +407,21 @@ jobs:
restore-keys: |
turbo-${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-
- run: pnpm install --frozen-lockfile
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: windows_suites
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.windows_suites }}
+ run: node scripts/ci/validate-execution-plan.mjs
- name: Run Windows process authority fixtures
+ if: contains(fromJSON(needs.scope.outputs.windows_suites), 'agent-driver')
run: pnpm --filter @alook/agent-driver exec vitest run src/internal/killTree.test.ts
- name: Run agent-driver tests in isolation
+ if: contains(fromJSON(needs.scope.outputs.windows_suites), 'agent-driver')
run: pnpm --filter @alook/agent-driver exec vitest run --no-file-parallelism --coverage --coverage.reporter=json --coverage.reporter=text
- name: Upload Windows agent-driver coverage
+ if: contains(fromJSON(needs.scope.outputs.windows_suites), 'agent-driver')
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
@@ -258,22 +430,28 @@ jobs:
fail_ci_if_error: true
name: windows-agent-driver
- name: Run Windows daemon admission fixture
+ if: contains(fromJSON(needs.scope.outputs.windows_suites), 'daemon')
run: |
pnpm --filter @alook/agent-driver build
pnpm --filter @alook/daemon exec vitest run src/daemon/createDaemon.test.ts -t "starts the builtin Codex lane from a fresh nested daemon workspace"
- name: Run daemon tests in isolation
+ if: contains(fromJSON(needs.scope.outputs.windows_suites), 'daemon')
env:
VITEST_MAX_WORKERS: 1
run: pnpm --filter @alook/daemon test
- - name: Run Windows-relevant workspace tests
- if: needs.scope.outputs.full == 'true'
- run: pnpm turbo run test --filter=@alook/cli --filter=@alook/app --filter=@alook/shared
- - name: Run affected Windows-relevant workspace tests
- if: needs.scope.outputs.full != 'true'
+ - name: Run selected Windows workspace tests
+ if: contains(fromJSON(needs.scope.outputs.windows_suites), 'app') || contains(fromJSON(needs.scope.outputs.windows_suites), 'cli') || contains(fromJSON(needs.scope.outputs.windows_suites), 'shared')
+ shell: bash
env:
- TURBO_SCM_BASE: ${{ needs.scope.outputs.base_sha }}
- TURBO_SCM_HEAD: ${{ needs.scope.outputs.head_sha }}
- run: pnpm turbo run test --affected --filter=@alook/cli --filter=@alook/app --filter=@alook/shared
+ WINDOWS_SUITES: ${{ needs.scope.outputs.windows_suites }}
+ run: |
+ filters=()
+ for suite in app cli shared; do
+ if node -e 'process.exit(JSON.parse(process.env.WINDOWS_SUITES).includes(process.argv[1]) ? 0 : 1)' "$suite"; then
+ filters+=("--filter=@alook/$suite")
+ fi
+ done
+ pnpm turbo run test "${filters[@]}"
app-packed-artifact:
name: App Packed Artifact
@@ -283,6 +461,13 @@ jobs:
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: run_app_packed_artifact
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.run_app_packed_artifact }}
+ run: node scripts/ci/validate-execution-plan.mjs
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
@@ -307,7 +492,7 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
- name: app-packed-artifact-${{ github.sha }}
+ name: app-packed-artifact-${{ needs.scope.outputs.head_sha }}
path: |
${{ runner.temp }}/alook-app-packed-artifact/*.tgz
${{ runner.temp }}/alook-app-packed-artifact/manifest.json
@@ -333,6 +518,13 @@ jobs:
node-version: 22
cache: pnpm
- run: pnpm install --frozen-lockfile
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: integration_suites
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.integration_suites }}
+ run: node scripts/ci/validate-execution-plan.mjs
- name: Create local bindings
run: printf 'BETTER_AUTH_SECRET=ci-e2e-secret\nBETTER_AUTH_URL=http://localhost:3000\nDEVICE_CLIENT_IDS=e2e-test-client\nENCRYPTION_KEY=ci-e2e-encryption-key-32chars!!\n' > src/web/.dev.vars
- run: pnpm run db:migrate
@@ -347,30 +539,30 @@ jobs:
wait_for() {
local url=$1 timeout=60 elapsed=0
until curl -sf "$url" > /dev/null 2>&1; do
- sleep 2
- elapsed=$((elapsed + 2))
+ sleep 2; elapsed=$((elapsed + 2))
if [ $elapsed -ge $timeout ]; then return 1; fi
done
}
- if ! wait_for http://localhost:8787/health email-worker || \
- ! wait_for http://localhost:8789/health ws-do || \
- ! wait_for http://localhost:3000/api/health web || \
- ! wait_for http://localhost:8790/health wake-worker; then
- pkill -f "wrangler" || true
- pkill -f "next" || true
- sleep 3
+ if ! wait_for http://localhost:8787/health || \
+ ! wait_for http://localhost:8789/health || \
+ ! wait_for http://localhost:3000/api/health || \
+ ! wait_for http://localhost:8790/health; then
+ pkill -f "wrangler" || true; pkill -f "next" || true; sleep 3
pnpm --filter @alook/email-worker dev &
pnpm --filter @alook/ws-do dev &
pnpm --filter @alook/web dev &
pnpm --filter @alook/wake-worker dev &
- wait_for http://localhost:8787/health email-worker
- wait_for http://localhost:8789/health ws-do
- wait_for http://localhost:3000/api/health web
- wait_for http://localhost:8790/health wake-worker
+ wait_for http://localhost:8787/health
+ wait_for http://localhost:8789/health
+ wait_for http://localhost:3000/api/health
+ wait_for http://localhost:8790/health
fi
- - run: pnpm test:e2e
- - run: pnpm --filter @alook/cli run test:integration
- - run: pnpm --filter @alook/daemon run test:integration
+ - if: contains(fromJSON(needs.scope.outputs.integration_suites), 'web')
+ run: pnpm test:e2e
+ - if: contains(fromJSON(needs.scope.outputs.integration_suites), 'cli')
+ run: pnpm --filter @alook/cli run test:integration
+ - if: contains(fromJSON(needs.scope.outputs.integration_suites), 'daemon')
+ run: pnpm --filter @alook/daemon run test:integration
desktop-rust:
name: Desktop Rust (test + machete)
@@ -385,6 +577,14 @@ jobs:
RUSTFLAGS: -D warnings
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ working-directory: ${{ github.workspace }}
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: run_rust
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.run_rust }}
+ run: node scripts/ci/validate-execution-plan.mjs
- uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
with:
@@ -407,6 +607,13 @@ jobs:
BETTER_AUTH_URL: http://localhost:3000
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: run_lighthouse
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.run_lighthouse }}
+ run: node scripts/ci/validate-execution-plan.mjs
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
@@ -417,6 +624,128 @@ jobs:
- working-directory: src/web
run: npx @lhci/cli autorun --config=lighthouserc.json && node scripts/print-lighthouse-scores.mjs
+ e2e-ui:
+ name: UI Playwright E2E (${{ matrix.shard }}/${{ matrix.total }})
+ needs: scope
+ if: needs.scope.outputs.run_ui_e2e == 'true' && github.event_name != 'push'
+ runs-on: ubuntu-latest
+ container:
+ image: ${{ matrix.image }}
+ options: --init --ipc=host --user 1001
+ defaults:
+ run:
+ shell: bash
+ timeout-minutes: 20
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJSON(needs.scope.outputs.e2e_matrix) }}
+ env:
+ BETTER_AUTH_SECRET: ci-e2e-ui-secret
+ BETTER_AUTH_URL: http://localhost:3000
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: e2e_matrix
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.e2e_matrix }}
+ run: node scripts/ci/validate-execution-plan.mjs
+ - env:
+ E2E_SPECS: ${{ toJSON(matrix.specs) }}
+ run: |
+ node scripts/ci/e2e-shard-artifacts.mjs write-manifest \
+ --output src/web/e2e-shard-manifest/shard-manifest.json \
+ --run-id "${{ github.run_id }}" \
+ --attempt "${{ github.run_attempt }}" \
+ --sha "${{ github.sha }}" \
+ --shard "${{ matrix.shard }}" \
+ --total "${{ matrix.total }}"
+ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 22
+ cache: pnpm
+ - run: pnpm install --frozen-lockfile
+ - run: printf 'BETTER_AUTH_SECRET=ci-e2e-ui-secret\nBETTER_AUTH_URL=http://localhost:3000\nDEVICE_CLIENT_IDS=e2e-test-client\nENCRYPTION_KEY=ci-e2e-encryption-key-32chars!!\nDEV_WS_DO_URL=http://localhost:3000\nNODE_ENV=development\n' > src/web/.dev.vars
+ - env:
+ E2E_SPECS: ${{ toJSON(matrix.specs) }}
+ run: |
+ mapfile -t specs < <(node -e 'for (const spec of JSON.parse(process.env.E2E_SPECS)) console.log(spec)')
+ pnpm --filter @alook/web exec playwright test "${specs[@]}"
+ - if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: blob-report-${{ github.run_id }}-${{ matrix.shard }}
+ path: |
+ src/web/blob-report/
+ src/web/e2e-shard-manifest/
+ if-no-files-found: error
+ overwrite: true
+ retention-days: 7
+ - if: failure()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: playwright-artifacts-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard }}
+ path: |
+ src/web/test-results/
+ src/web/e2e-service-logs/
+ if-no-files-found: ignore
+ retention-days: 7
+
+ merge-reports:
+ name: Merge Playwright Reports
+ needs: [scope, e2e-ui]
+ if: ${{ !cancelled() && needs.scope.outputs.run_ui_e2e == 'true' && github.event_name != 'push' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ CI_PROJECTION_NAME: e2e_matrix
+ CI_PROJECTION_VALUE: ${{ needs.scope.outputs.e2e_matrix }}
+ run: node scripts/ci/validate-execution-plan.mjs
+ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 22
+ cache: pnpm
+ - run: pnpm install --frozen-lockfile
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ pattern: blob-report-${{ github.run_id }}-*
+ path: downloaded-blob-reports
+ merge-multiple: false
+ - env:
+ E2E_MATRIX: ${{ needs.scope.outputs.e2e_matrix }}
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ node scripts/ci/e2e-shard-artifacts.mjs verify-artifacts \
+ --root downloaded-blob-reports \
+ --output verified-blob-reports \
+ --run-id "${{ github.run_id }}" \
+ --attempt "${{ github.run_attempt }}" \
+ --sha "${{ github.sha }}"
+ - env:
+ PLAYWRIGHT_HTML_OUTPUT_DIR: ${{ github.workspace }}/playwright-report
+ PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/merged-report.json
+ run: pnpm --filter @alook/web exec playwright merge-reports --reporter html,json "${{ github.workspace }}/verified-blob-reports"
+ - env:
+ E2E_MATRIX: ${{ needs.scope.outputs.e2e_matrix }}
+ run: node scripts/ci/e2e-shard-artifacts.mjs verify-merged --report merged-report.json
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: playwright-report-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ playwright-report/
+ merged-report.json
+ if-no-files-found: error
+ retention-days: 14
+
ci-gate:
name: CI Gate
if: always()
@@ -437,8 +766,8 @@ jobs:
[
{"name":"scope","expected":true,"result":"${{ needs.scope.result }}"},
{"name":"blog-build","expected":${{ needs.scope.outputs.run_blog_build == 'true' }},"result":"${{ needs.blog-build.result }}"},
- {"name":"static-checks","expected":${{ needs.scope.outputs.run_code_checks == 'true' }},"result":"${{ needs.static-checks.result }}"},
- {"name":"test-linux","expected":${{ needs.scope.outputs.run_code_checks == 'true' }},"result":"${{ needs.test-linux.result }}"},
+ {"name":"static-checks","expected":${{ needs.scope.outputs.run_static_checks == 'true' }},"result":"${{ needs.static-checks.result }}"},
+ {"name":"test-linux","expected":${{ needs.scope.outputs.run_linux == 'true' }},"result":"${{ needs.test-linux.result }}"},
{"name":"test-windows","expected":${{ needs.scope.outputs.run_windows == 'true' }},"result":"${{ needs.test-windows.result }}"},
{"name":"app-packed-artifact","expected":${{ needs.scope.outputs.run_app_packed_artifact == 'true' }},"result":"${{ needs.app-packed-artifact.result }}"},
{"name":"e2e","expected":${{ needs.scope.outputs.run_e2e == 'true' && github.event_name != 'push' }},"result":"${{ needs.e2e.result }}"},
@@ -447,4 +776,31 @@ jobs:
]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ run: node scripts/ci/validate-execution-plan.mjs
+ - run: node scripts/ci/assert-gate.mjs
+
+ ui-e2e-gate:
+ name: UI E2E Gate
+ if: always()
+ needs: [scope, e2e-ui, merge-reports]
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ env:
+ CI_GATE_RESULTS: >-
+ [
+ {"name":"scope","expected":true,"result":"${{ needs.scope.result }}"},
+ {"name":"e2e-ui","expected":${{ needs.scope.outputs.run_ui_e2e == 'true' && github.event_name != 'push' }},"result":"${{ needs.e2e-ui.result }}"},
+ {"name":"merge-reports","expected":${{ needs.scope.outputs.run_ui_e2e == 'true' && github.event_name != 'push' }},"result":"${{ needs.merge-reports.result }}"}
+ ]
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Validate execution plan
+ env:
+ CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}
+ CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}
+ run: node scripts/ci/validate-execution-plan.mjs
- run: node scripts/ci/assert-gate.mjs
diff --git a/.github/workflows/e2e-ui.yml b/.github/workflows/e2e-ui.yml
deleted file mode 100644
index c5d9f2baa..000000000
--- a/.github/workflows/e2e-ui.yml
+++ /dev/null
@@ -1,164 +0,0 @@
-name: E2E UI
-
-on:
- pull_request:
- branches: [main]
- merge_group:
- types: [checks_requested]
- workflow_dispatch:
-
-permissions:
- contents: read
- actions: read
-
-concurrency:
- group: e2e-ui-${{ github.event.pull_request.number || github.event.merge_group.head_sha || github.ref }}
- cancel-in-progress: true
-
-jobs:
- scope:
- name: UI Change Scope
- runs-on: ubuntu-latest
- timeout-minutes: 5
- outputs:
- run_ui_e2e: ${{ steps.scope.outputs.run_ui_e2e }}
- e2e_matrix: ${{ steps.scope.outputs.e2e_matrix }}
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- fetch-depth: 0
- - id: scope
- env:
- BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.event.before }}
- HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
- FORCE_FULL: ${{ github.event_name == 'workflow_dispatch' }}
- run: |
- args=(--base "$BASE_SHA" --head "$HEAD_SHA" --output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY")
- if [[ "$FORCE_FULL" == "true" ]]; then args+=(--force-full); fi
- node scripts/ci/changed-scopes.mjs "${args[@]}"
- node scripts/ci/e2e-shards.mjs --output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
-
- e2e-ui:
- name: UI Playwright E2E (${{ matrix.shard }}/${{ matrix.total }})
- needs: scope
- if: needs.scope.outputs.run_ui_e2e == 'true'
- runs-on: ubuntu-latest
- container:
- image: ${{ matrix.image }}
- options: --init --ipc=host --user 1001
- defaults:
- run:
- shell: bash
- timeout-minutes: 20
- strategy:
- fail-fast: false
- matrix: ${{ fromJSON(needs.scope.outputs.e2e_matrix) }}
- env:
- BETTER_AUTH_SECRET: ci-e2e-ui-secret
- BETTER_AUTH_URL: http://localhost:3000
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - env:
- E2E_SPECS: ${{ toJSON(matrix.specs) }}
- run: |
- node scripts/ci/e2e-shard-artifacts.mjs write-manifest \
- --output src/web/e2e-shard-manifest/shard-manifest.json \
- --run-id "${{ github.run_id }}" \
- --attempt "${{ github.run_attempt }}" \
- --sha "${{ github.sha }}" \
- --shard "${{ matrix.shard }}" \
- --total "${{ matrix.total }}"
- - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
- with:
- node-version: 22
- cache: pnpm
- - run: pnpm install --frozen-lockfile
- - run: printf 'BETTER_AUTH_SECRET=ci-e2e-ui-secret\nBETTER_AUTH_URL=http://localhost:3000\nDEVICE_CLIENT_IDS=e2e-test-client\nENCRYPTION_KEY=ci-e2e-encryption-key-32chars!!\nDEV_WS_DO_URL=http://localhost:3000\nNODE_ENV=development\n' > src/web/.dev.vars
- - env:
- E2E_SPECS: ${{ toJSON(matrix.specs) }}
- run: |
- mapfile -t specs < <(node -e 'for (const spec of JSON.parse(process.env.E2E_SPECS)) console.log(spec)')
- pnpm --filter @alook/web exec playwright test "${specs[@]}"
- - if: ${{ !cancelled() }}
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
- with:
- name: blob-report-${{ github.run_id }}-${{ matrix.shard }}
- path: |
- src/web/blob-report/
- src/web/e2e-shard-manifest/
- if-no-files-found: error
- overwrite: true
- retention-days: 7
- - if: failure()
- continue-on-error: true
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
- with:
- name: playwright-artifacts-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard }}
- path: |
- src/web/test-results/
- src/web/e2e-service-logs/
- if-no-files-found: ignore
- retention-days: 7
-
- merge-reports:
- name: Merge Playwright Reports
- needs: [scope, e2e-ui]
- if: ${{ !cancelled() && needs.scope.outputs.run_ui_e2e == 'true' }}
- runs-on: ubuntu-latest
- timeout-minutes: 5
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
- with:
- node-version: 22
- cache: pnpm
- - run: pnpm install --frozen-lockfile
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
- with:
- pattern: blob-report-${{ github.run_id }}-*
- path: downloaded-blob-reports
- merge-multiple: false
- - env:
- E2E_MATRIX: ${{ needs.scope.outputs.e2e_matrix }}
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- node scripts/ci/e2e-shard-artifacts.mjs verify-artifacts \
- --root downloaded-blob-reports \
- --output verified-blob-reports \
- --run-id "${{ github.run_id }}" \
- --attempt "${{ github.run_attempt }}" \
- --sha "${{ github.sha }}"
- - env:
- PLAYWRIGHT_HTML_OUTPUT_DIR: ${{ github.workspace }}/playwright-report
- PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/merged-report.json
- run: pnpm --filter @alook/web exec playwright merge-reports --reporter html,json "${{ github.workspace }}/verified-blob-reports"
- - env:
- E2E_MATRIX: ${{ needs.scope.outputs.e2e_matrix }}
- run: node scripts/ci/e2e-shard-artifacts.mjs verify-merged --report merged-report.json
- - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
- with:
- name: playwright-report-${{ github.run_id }}-${{ github.run_attempt }}
- path: |
- playwright-report/
- merged-report.json
- if-no-files-found: error
- retention-days: 14
-
- ui-e2e-gate:
- name: UI E2E Gate
- if: always()
- needs: [scope, e2e-ui, merge-reports]
- runs-on: ubuntu-latest
- timeout-minutes: 5
- env:
- CI_GATE_RESULTS: >-
- [
- {"name":"scope","expected":true,"result":"${{ needs.scope.result }}"},
- {"name":"e2e-ui","expected":${{ needs.scope.outputs.run_ui_e2e == 'true' }},"result":"${{ needs.e2e-ui.result }}"},
- {"name":"merge-reports","expected":${{ needs.scope.outputs.run_ui_e2e == 'true' }},"result":"${{ needs.merge-reports.result }}"}
- ]
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - run: node scripts/ci/assert-gate.mjs
diff --git a/scripts/ci/change-scope-manifest.json b/scripts/ci/change-scope-manifest.json
new file mode 100644
index 000000000..c4d39205a
--- /dev/null
+++ b/scripts/ci/change-scope-manifest.json
@@ -0,0 +1,231 @@
+{
+ "schema_version": 1,
+ "policy_version": 1,
+ "packages": [
+ {
+ "name": "@alook/agent-driver",
+ "root": "src/daemon/agent-driver",
+ "static": true,
+ "build": false,
+ "knip": true,
+ "unit_root": "src/daemon/agent-driver",
+ "coverage_roots": ["src/daemon/agent-driver/**/*.{ts,tsx,js,jsx}"],
+ "windows_suites": ["agent-driver"],
+ "integration_suites": [],
+ "linux_suites": ["agent-driver-packed"],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": true
+ },
+ {
+ "name": "@alook/app",
+ "root": "src/app",
+ "static": true,
+ "build": false,
+ "knip": true,
+ "unit_root": "src/app",
+ "coverage_roots": ["src/app/**/*.{ts,tsx,js,jsx}"],
+ "windows_suites": ["app"],
+ "integration_suites": [],
+ "linux_suites": [],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": true
+ },
+ {
+ "name": "@alook/cli",
+ "root": "src/cli",
+ "static": true,
+ "build": true,
+ "knip": true,
+ "unit_root": "src/cli",
+ "coverage_roots": ["src/cli/**/*.{ts,tsx,js,jsx}"],
+ "codecov_target": "cli",
+ "windows_suites": ["cli"],
+ "integration_suites": ["cli"],
+ "linux_suites": [],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": false
+ },
+ {
+ "name": "@alook/daemon",
+ "root": "src/daemon",
+ "static": true,
+ "build": false,
+ "knip": false,
+ "unit_root": "src/daemon",
+ "coverage_roots": [
+ "src/daemon/scripts/**/*.{ts,tsx,js,jsx}",
+ "src/daemon/src/**/*.{ts,tsx,js,jsx}"
+ ],
+ "windows_suites": ["daemon"],
+ "integration_suites": ["daemon"],
+ "linux_suites": ["daemon-packed", "daemon-real"],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": true
+ },
+ {
+ "name": "@alook/desktop",
+ "root": "src/desktop",
+ "static": false,
+ "build": false,
+ "knip": false,
+ "unit_root": null,
+ "coverage_roots": [],
+ "windows_suites": [],
+ "integration_suites": [],
+ "linux_suites": [],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": false,
+ "rust": true
+ },
+ {
+ "name": "@alook/email-worker",
+ "root": "src/email-worker",
+ "static": true,
+ "build": true,
+ "knip": true,
+ "unit_root": "src/email-worker",
+ "coverage_roots": ["src/email-worker/**/*.{ts,tsx,js,jsx}"],
+ "codecov_target": "email-worker",
+ "windows_suites": [],
+ "integration_suites": ["web"],
+ "linux_suites": [],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": true
+ },
+ {
+ "name": "@alook/shared",
+ "root": "src/shared",
+ "static": true,
+ "build": true,
+ "knip": true,
+ "unit_root": "src/shared",
+ "coverage_roots": ["src/shared/**/*.{ts,tsx,js,jsx}"],
+ "codecov_target": "shared",
+ "windows_suites": ["shared"],
+ "integration_suites": [],
+ "linux_suites": [],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": true
+ },
+ {
+ "name": "@alook/test-utils",
+ "root": "tests/utils",
+ "static": true,
+ "build": false,
+ "knip": false,
+ "unit_root": "tests/utils",
+ "coverage_roots": [],
+ "windows_suites": [],
+ "integration_suites": [],
+ "linux_suites": [],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": false
+ },
+ {
+ "name": "@alook/wake-worker",
+ "root": "src/wake-worker",
+ "static": true,
+ "build": true,
+ "knip": true,
+ "unit_root": "src/wake-worker",
+ "coverage_roots": ["src/wake-worker/**/*.{ts,tsx,js,jsx}"],
+ "windows_suites": [],
+ "integration_suites": ["web"],
+ "linux_suites": [],
+ "ui": false,
+ "lighthouse": false,
+ "app_artifact": true
+ },
+ {
+ "name": "@alook/web",
+ "root": "src/web",
+ "static": true,
+ "build": true,
+ "knip": true,
+ "unit_root": "src/web",
+ "coverage_roots": ["src/web/**/*.{ts,tsx,js,jsx}"],
+ "codecov_target": "web",
+ "windows_suites": [],
+ "integration_suites": ["web"],
+ "linux_suites": [],
+ "ui": true,
+ "lighthouse": true,
+ "app_artifact": true
+ },
+ {
+ "name": "@alook/ws-do",
+ "root": "src/ws-do",
+ "static": true,
+ "build": true,
+ "knip": true,
+ "unit_root": "src/ws-do",
+ "coverage_roots": ["src/ws-do/**/*.{ts,tsx,js,jsx}"],
+ "codecov_target": "ws-do",
+ "windows_suites": [],
+ "integration_suites": ["web"],
+ "linux_suites": [],
+ "ui": true,
+ "lighthouse": false,
+ "app_artifact": true
+ }
+ ],
+ "suites": {
+ "windows": ["agent-driver", "app", "cli", "daemon", "shared"],
+ "integration": ["cli", "daemon", "web"],
+ "linux": ["agent-driver-packed", "daemon-packed", "daemon-real"]
+ },
+ "path_suites": [
+ { "root": "tests/integration/cli", "integration_suites": ["cli"] },
+ { "root": "tests/integration/daemon", "integration_suites": ["daemon"] }
+ ],
+ "ui": {
+ "root": "src/web/src/test/e2e-ui",
+ "all": "all",
+ "contracts": {
+ "blog": ["54-blog-multizone.spec.ts"]
+ }
+ },
+ "codecov_targets": {
+ "shared": { "path": "src/shared", "target": 50 },
+ "cli": { "path": "src/cli", "target": 60 },
+ "email-worker": { "path": "src/email-worker", "target": 70 },
+ "ws-do": { "path": "src/ws-do", "target": 70 },
+ "web": { "path": "src/web", "target": 30 }
+ },
+ "integrity_paths": [
+ ".github/workflows/ci.yml",
+ "codecov.yml",
+ "package.json",
+ "pnpm-lock.yaml",
+ "pnpm-workspace.yaml",
+ "turbo.json",
+ "vitest.config.ts",
+ "scripts/ci/assert-gate.mjs",
+ "scripts/ci/change-scope-manifest.json",
+ "scripts/ci/changed-scopes.mjs",
+ "scripts/ci/characterization-guard.mjs",
+ "scripts/ci/e2e-shard-artifacts.mjs",
+ "scripts/ci/e2e-shards.mjs",
+ "scripts/ci/verify-coverage-plan.mjs",
+ "scripts/ci/validate-execution-plan.mjs",
+ "src/app/package.json",
+ "src/cli/package.json",
+ "src/daemon/agent-driver/package.json",
+ "src/daemon/package.json",
+ "src/desktop/package.json",
+ "src/email-worker/package.json",
+ "src/shared/package.json",
+ "src/wake-worker/package.json",
+ "src/web/package.json",
+ "src/ws-do/package.json",
+ "tests/utils/package.json"
+ ]
+}
diff --git a/scripts/ci/changed-scopes.mjs b/scripts/ci/changed-scopes.mjs
index d313990ff..17d0462d7 100644
--- a/scripts/ci/changed-scopes.mjs
+++ b/scripts/ci/changed-scopes.mjs
@@ -1,22 +1,23 @@
import { execFileSync } from "node:child_process"
-import { appendFileSync, readFileSync } from "node:fs"
+import { createHash } from "node:crypto"
+import {
+ appendFileSync,
+ readFileSync,
+ writeFileSync,
+} from "node:fs"
import { resolve } from "node:path"
import { fileURLToPath } from "node:url"
+import { createE2eMatrix, discoverE2eSpecs } from "./e2e-shards.mjs"
+const ROOT = resolve(fileURLToPath(new URL("../..", import.meta.url)))
+const MANIFEST_URL = new URL("./change-scope-manifest.json", import.meta.url)
const BLOG_ROOT = /^src\/web\/blog(?:\/|$)/
const BLOG_CONTENT = /^src\/web\/blog\/src\/content\/[^/]+\.mdx$/
const BLOG_ASSET = /^src\/web\/blog\/public\/blog(?:\/|$)/
-const BLOG_SHARED_INPUTS = new Set(
- JSON.parse(
- readFileSync(
- new URL("../../src/web/blog/shared-build-inputs.json", import.meta.url),
- "utf8",
- ),
- ).sourcePaths.map((path) => `src/web/${path}`),
-)
const MARKDOWN = /(?:^|\/)\w[^/]*\.md$/i
const WORKFLOW = /^\.github\/workflows\//
const GLOBAL_PATHS = new Set([
+ "codecov.yml",
"package.json",
"pnpm-lock.yaml",
"pnpm-workspace.yaml",
@@ -24,207 +25,600 @@ const GLOBAL_PATHS = new Set([
"tsconfig.json",
"vitest.config.ts",
])
-const KNOWN_ROOTS = [
- ".claude/",
- ".openai/",
- "docs/",
- "src/app/",
- "src/cli/",
- "src/daemon/",
- "src/desktop/",
- "src/email-worker/",
- "src/shared/",
- "src/wake-worker/",
- "src/web/",
- "src/ws-do/",
- "tests/integration/cli/",
- "tests/integration/daemon/",
- "tests/utils/",
+const KNOWN_PREFIXES = [".claude/", ".openai/", "docs/"]
+const COVERAGE_EXCLUDES = [
+ /(?:^|\/)\w[^/]*\.(?:test|spec)\.[cm]?[jt]sx?$/,
+ /(?:^|\/)node_modules\//,
+ /(?:^|\/)(?:\.next|\.open-next|\.wrangler|dist|bundled|__mocks__)\//,
+ /(?:^|\/)test-runtime\//,
+ /(?:^|\/)test-harness\.ts$/,
+ /\.d\.ts$/,
+ /^src\/cli\/src\/index\.ts$/,
+ /^src\/web\/scripts\//,
+ /^src\/web\/readme-capture\//,
+ /^src\/web\/src\/.*\.tsx$/,
+ /^src\/web\/src\/test\/fixtures\//,
+ /^src\/web\/src\/hooks\/(?:use-agent-chat|use-chat-sheets|use-file-attachments|use-message-flags|use-text-selection-quote)\.ts$/,
+ /^src\/web\/src\/components\/agent-chat\/use-rotating-placeholder\.ts$/,
]
function normalizePath(path) {
- return path.replaceAll("\\", "/").replace(/^\.\//, "")
+ return path.replaceAll("\\", "/").replace(/^\.\//, "").replace(/\/$/, "")
}
-function isBlogPath(path) {
+function sorted(values) {
+ return [...new Set(values.filter((value) => value !== null && value !== undefined))].sort()
+}
+
+function stableValue(value) {
+ if (Array.isArray(value)) return value.map(stableValue)
+ if (value && typeof value === "object") {
+ return Object.fromEntries(
+ Object.keys(value).sort().map((key) => [key, stableValue(value[key])]),
+ )
+ }
+ return value
+}
+
+export function stablePlanJson(plan) {
+ return JSON.stringify(stableValue(plan))
+}
+
+function hashPlan(plan) {
+ const { plan_hash: _ignored, ...unsigned } = plan
+ return createHash("sha256").update(stablePlanJson(unsigned)).digest("hex")
+}
+
+function isBlogContent(path) {
return BLOG_CONTENT.test(path) || BLOG_ASSET.test(path)
}
-function isBlogBuildInput(path) {
- return BLOG_ROOT.test(path) || [...BLOG_SHARED_INPUTS].some((input) => (
+function isMarkdown(path) {
+ return MARKDOWN.test(path)
+}
+
+function pathWithin(path, root) {
+ return path === root || path.startsWith(`${root}/`)
+}
+
+function coverageRootPrefix(glob) {
+ return glob.split("/**")[0]
+}
+
+function coverageRootMatches(path, glob) {
+ if (!pathWithin(path, coverageRootPrefix(glob))) return false
+ const extensionSet = glob.match(/\.\{([^}]+)\}$/)?.[1]
+ if (extensionSet) {
+ return extensionSet.split(",").some((extension) => path.endsWith(`.${extension}`))
+ }
+ const extension = glob.match(/\.([0-9A-Za-z]+)$/)?.[1]
+ return extension ? path.endsWith(`.${extension}`) : false
+}
+
+function isCoverable(path, coverageRoots) {
+ return coverageRoots.some((root) => coverageRootMatches(path, root))
+ && COVERAGE_EXCLUDES.every((pattern) => !pattern.test(path))
+}
+
+function changePaths(change) {
+ return change.old_path ? [change.old_path, change.path] : [change.path]
+}
+
+function normalizedChanges(changes) {
+ return changes
+ .map((change) => ({
+ status: String(change.status || "M"),
+ ...(change.old_path ? { old_path: normalizePath(change.old_path) } : {}),
+ path: normalizePath(change.path),
+ }))
+ .filter((change) => change.path)
+ .sort((left, right) => (
+ left.path.localeCompare(right.path)
+ || (left.old_path || "").localeCompare(right.old_path || "")
+ || left.status.localeCompare(right.status)
+ ))
+}
+
+export function loadScopeManifest(path = MANIFEST_URL) {
+ return JSON.parse(readFileSync(path, "utf8"))
+}
+
+function packageManifest(scopePackage, root = ROOT) {
+ return JSON.parse(readFileSync(resolve(root, scopePackage.root, "package.json"), "utf8"))
+}
+
+function workspaceGraph(manifest, root = ROOT) {
+ const names = new Set(manifest.packages.map((entry) => entry.name))
+ const dependents = new Map(manifest.packages.map((entry) => [entry.name, new Set()]))
+ for (const entry of manifest.packages) {
+ const workspacePackage = packageManifest(entry, root)
+ const dependencies = {
+ ...workspacePackage.dependencies,
+ ...workspacePackage.devDependencies,
+ ...workspacePackage.optionalDependencies,
+ ...workspacePackage.peerDependencies,
+ }
+ for (const [dependency, version] of Object.entries(dependencies)) {
+ if (names.has(dependency) && String(version).startsWith("workspace:")) {
+ dependents.get(dependency).add(entry.name)
+ }
+ }
+ }
+ return dependents
+}
+
+function validateCodecovTargets(manifest, root) {
+ const source = readFileSync(resolve(root, "codecov.yml"), "utf8").replaceAll("\r\n", "\n")
+ const projectSection = source.match(/^[ ]{4}project:\n([\s\S]*?)^[ ]{4}patch:/m)?.[1]
+ if (!projectSection) throw new Error("codecov.yml project status section is missing")
+ const configuredNames = [...projectSection.matchAll(/^[ ]{6}([^\s:\n][^:\n]*):$/gm)]
+ .map((match) => match[1])
+ .sort()
+ const manifestNames = Object.keys(manifest.codecov_targets).sort()
+ if (JSON.stringify(configuredNames) !== JSON.stringify(manifestNames)) {
+ throw new Error("scope manifest Codecov project target names do not match codecov.yml")
+ }
+ for (const [name, target] of Object.entries(manifest.codecov_targets)) {
+ const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+ const escapedPath = target.path.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+ const contract = new RegExp(
+ `^[ ]{6}${escapedName}:\\n[ ]{8}target: ${target.target}%\\n[ ]{8}paths:\\n[ ]{10}- ${escapedPath}/\\*\\*$`,
+ "m",
+ )
+ if (!contract.test(projectSection)) {
+ throw new Error(`scope manifest Codecov target does not match codecov.yml: ${name}`)
+ }
+ }
+}
+
+export function validateScopeManifest(manifest, options = {}) {
+ const root = options.root || ROOT
+ if (manifest.schema_version !== 1 || manifest.policy_version !== 1) {
+ throw new Error("scope manifest schema/policy version must be 1")
+ }
+ if (!Array.isArray(manifest.packages) || manifest.packages.length === 0) {
+ throw new Error("scope manifest packages are required")
+ }
+
+ const names = manifest.packages.map((entry) => entry.name)
+ const roots = manifest.packages.map((entry) => normalizePath(entry.root))
+ if (new Set(names).size !== names.length) throw new Error("package names must be unique")
+ if (new Set(roots).size !== roots.length) throw new Error("package root must be unique")
+ const workspaceRoots = [...readFileSync(resolve(root, "pnpm-workspace.yaml"), "utf8")
+ .matchAll(/^[ ]{2}- ["']([^"']+)["']$/gm)]
+ .map((match) => normalizePath(match[1]))
+ .sort()
+ if (JSON.stringify([...roots].sort()) !== JSON.stringify(workspaceRoots)) {
+ throw new Error("scope manifest package roots must exactly match pnpm-workspace.yaml")
+ }
+
+ for (const [kind, registry] of Object.entries(manifest.suites)) {
+ if (new Set(registry).size !== registry.length) throw new Error(`${kind} suite IDs must be unique`)
+ for (const entry of manifest.packages) {
+ for (const suite of entry[`${kind}_suites`]) {
+ if (!registry.includes(suite)) throw new Error(`unknown ${kind} suite: ${suite}`)
+ }
+ }
+ const declared = sorted(manifest.packages.flatMap((entry) => entry[`${kind}_suites`]))
+ if (JSON.stringify([...registry].sort()) !== JSON.stringify(declared)) {
+ throw new Error(`${kind} suite registry must exactly match package declarations`)
+ }
+ }
+ const pathSuiteRoots = manifest.path_suites.map((entry) => normalizePath(entry.root))
+ if (new Set(pathSuiteRoots).size !== pathSuiteRoots.length) {
+ throw new Error("path suite roots must be unique")
+ }
+ for (const entry of manifest.path_suites) {
+ for (const suite of entry.integration_suites) {
+ if (!manifest.suites.integration.includes(suite)) {
+ throw new Error(`unknown path integration suite: ${suite}`)
+ }
+ }
+ }
+
+ for (const entry of manifest.packages) {
+ const workspacePackage = packageManifest(entry, root)
+ if (workspacePackage.name !== entry.name) {
+ throw new Error(`package manifest mismatch for ${entry.root}`)
+ }
+ if (entry.codecov_target && !manifest.codecov_targets[entry.codecov_target]) {
+ throw new Error(`unknown Codecov target: ${entry.codecov_target}`)
+ }
+ for (const script of entry.unit_root ? ["test"] : []) {
+ if (!workspacePackage.scripts?.[script]) {
+ throw new Error(`${entry.name} declares ${script} scope without a package script`)
+ }
+ }
+ }
+
+ const specs = options.specs || discoverE2eSpecs(resolve(root, manifest.ui.root))
+ for (const contractSpecs of Object.values(manifest.ui.contracts)) {
+ for (const spec of contractSpecs) {
+ if (!specs.includes(spec)) throw new Error(`unknown UI spec: ${spec}`)
+ }
+ }
+ validateCodecovTargets(manifest, root)
+ sorted(manifest.integrity_paths)
+ workspaceGraph(manifest, root)
+ return manifest
+}
+
+function findPackage(path, manifest) {
+ return [...manifest.packages]
+ .sort((left, right) => right.root.length - left.root.length)
+ .find((entry) => pathWithin(path, entry.root))
+}
+
+function affectedClosure(directNames, manifest) {
+ const graph = workspaceGraph(manifest)
+ const affected = new Set(directNames)
+ const queue = [...directNames]
+ while (queue.length > 0) {
+ const current = queue.shift()
+ for (const dependent of graph.get(current) || []) {
+ if (affected.has(dependent)) continue
+ affected.add(dependent)
+ queue.push(dependent)
+ }
+ }
+ return sorted([...affected])
+}
+
+function loadBlogSharedInputs() {
+ const sourcePaths = JSON.parse(
+ readFileSync(resolve(ROOT, "src/web/blog/shared-build-inputs.json"), "utf8"),
+ ).sourcePaths
+ return sourcePaths.map((path) => `src/web/${path}`)
+}
+
+function isBlogBuildInput(path, sharedInputs) {
+ return BLOG_ROOT.test(path) || sharedInputs.some((input) => (
input.endsWith("/") ? path.startsWith(input) : path === input
))
}
-function isMarkdownPath(path) {
- return MARKDOWN.test(path)
+function isKnownPath(path, manifest) {
+ return isMarkdown(path)
+ || GLOBAL_PATHS.has(path)
+ || manifest.packages.some((entry) => pathWithin(path, entry.root))
+ || manifest.path_suites.some((entry) => pathWithin(path, normalizePath(entry.root)))
+ || KNOWN_PREFIXES.some((prefix) => path.startsWith(prefix))
}
-function isKnownPath(path) {
- return (
- isMarkdownPath(path) ||
- isBlogPath(path) ||
- GLOBAL_PATHS.has(path) ||
- KNOWN_ROOTS.some((root) => path.startsWith(root))
- )
+function fullReason(paths, { forceFull, fallbackReason }) {
+ if (fallbackReason) return "classifier_error"
+ if (forceFull) return "forced"
+ if (paths.length === 0) return "empty_diff"
+ const docsOnly = paths.every(isMarkdown)
+ const blogOnly = paths.every(isBlogContent)
+ if (!docsOnly && !blogOnly && paths.every((path) => isMarkdown(path) || isBlogContent(path))) {
+ return "mixed_content"
+ }
+ if (paths.some((path) => WORKFLOW.test(path) || GLOBAL_PATHS.has(path) || path.startsWith("scripts/") || (path.startsWith(".github/") && !isMarkdown(path)))) {
+ return "policy_change"
+ }
+ if (paths.some((path) => [".claude/", ".openai/", "docs/"]
+ .some((prefix) => path.startsWith(prefix)) && !isMarkdown(path))) return "policy_change"
+ return null
}
-export function classifyPaths(inputPaths, options = {}) {
- const paths = [...new Set(inputPaths.map(normalizePath).filter(Boolean))].sort()
- const empty = paths.length === 0
- const docsOnly = !empty && paths.every(isMarkdownPath)
- const blogOnly = !empty && paths.every(isBlogPath)
- const contentMix =
- !docsOnly && !blogOnly && paths.every((path) => isMarkdownPath(path) || isBlogPath(path))
- const workflowChanged = paths.some((path) => WORKFLOW.test(path))
- const globalChanged = paths.some(
- (path) =>
- GLOBAL_PATHS.has(path) ||
- path.startsWith("scripts/") ||
- (path.startsWith(".github/") && !isMarkdownPath(path))
- )
- const unknownChanged = paths.some((path) => !isKnownPath(path))
- const forceFull = options.forceFull === true
- const full = empty || forceFull || contentMix || workflowChanged || globalChanged || unknownChanged
- const effectiveDocsOnly = docsOnly && !full
- const effectiveBlogOnly = blogOnly && !full
- const blogCodeOnly =
- !empty &&
- paths.every(
- (path) =>
- BLOG_ROOT.test(path) &&
- (!path.startsWith("src/web/blog/src/content/") || BLOG_CONTENT.test(path)),
- )
- const web = full || paths.some((path) => path.startsWith("src/web/"))
- const shared = full || paths.some((path) => path.startsWith("src/shared/"))
- const cli = full || paths.some((path) => path.startsWith("src/cli/"))
- const daemon = full || paths.some((path) => path.startsWith("src/daemon/"))
- const desktop = full || paths.some((path) => path.startsWith("src/desktop/"))
- const wsDo = full || paths.some((path) => path.startsWith("src/ws-do/"))
- const worker =
- full ||
- paths.some(
- (path) =>
- path.startsWith("src/email-worker/") ||
- path.startsWith("src/wake-worker/") ||
- path.startsWith("src/ws-do/")
- )
- const app = full || paths.some((path) => path.startsWith("src/app/"))
- const integration = full || paths.some((path) => path.startsWith("tests/integration/"))
- const codeChanged =
- full || paths.some((path) => !isMarkdownPath(path) && !isBlogPath(path))
- const runCodeChecks = codeChanged && !effectiveBlogOnly && !effectiveDocsOnly
- const appPackedArtifactInput = paths.some(
- (path) =>
- path.startsWith("src/app/") ||
- path.startsWith("src/daemon/") ||
- path.startsWith("src/shared/") ||
- path.startsWith("src/email-worker/") ||
- path.startsWith("src/ws-do/") ||
- path.startsWith("src/wake-worker/") ||
- (path.startsWith("src/web/") && !BLOG_ROOT.test(path))
- )
+function emptyJobs() {
+ return {
+ app_packed_artifact: false,
+ blog_build: false,
+ e2e: false,
+ lighthouse: false,
+ merge_reports: false,
+ rust: false,
+ static_checks: false,
+ test_linux: false,
+ test_windows: false,
+ ui_e2e: false,
+ }
+}
+
+function packageSets(packages) {
+ return {
+ static: sorted(packages.filter((entry) => entry.static).map((entry) => entry.name)),
+ build: sorted(packages.filter((entry) => entry.build).map((entry) => entry.name)),
+ knip: sorted(packages.filter((entry) => entry.knip).map((entry) => entry.name)),
+ unit: sorted(packages.map((entry) => entry.unit_root).filter(Boolean)),
+ windows: sorted(packages.flatMap((entry) => entry.windows_suites)),
+ integration: sorted(packages.flatMap((entry) => entry.integration_suites)),
+ linux: sorted(packages.flatMap((entry) => entry.linux_suites)),
+ }
+}
+function coverageSets(packages) {
return {
+ targets: sorted(packages.map((entry) => entry.codecov_target).filter(Boolean)),
+ include_roots: sorted(packages.flatMap((entry) => entry.coverage_roots)),
+ }
+}
+
+function requiredCoverageFiles(changes, coverageRoots) {
+ return sorted(changes.flatMap((change) => {
+ const status = change.status[0]
+ if (!["A", "M", "R", "C"].includes(status)) return []
+ return isCoverable(change.path, coverageRoots) ? [change.path] : []
+ }))
+}
+
+function planClass({ full, docsOnly, blogPathsOnly, direct }) {
+ if (full) return "full"
+ if (docsOnly) return "docs"
+ if (blogPathsOnly) return "blog"
+ if (direct.length === 1 && direct[0] === "@alook/shared") return "shared"
+ if (direct.length === 1) return "package"
+ return "mixed"
+}
+
+export function buildExecutionPlan(inputChanges, options = {}) {
+ const manifest = validateScopeManifest(options.manifest || loadScopeManifest())
+ const changes = normalizedChanges(inputChanges)
+ const paths = sorted(changes.flatMap(changePaths))
+ const docsOnly = paths.length > 0 && paths.every(isMarkdown)
+ const blogContentOnly = paths.length > 0 && paths.every(isBlogContent)
+ const blogPathsOnly = paths.length > 0 && paths.every((path) => BLOG_ROOT.test(path))
+ const unknown = paths.some((path) => !isKnownPath(path, manifest))
+ const reason = fullReason(paths, options) || (unknown ? "unknown_path" : null)
+ const full = reason !== null
+ const direct = sorted(paths.map((path) => findPackage(path, manifest)?.name).filter(Boolean))
+
+ let affectedNames
+ if (full) affectedNames = sorted(manifest.packages.map((entry) => entry.name))
+ else if (docsOnly || blogContentOnly) affectedNames = []
+ else affectedNames = affectedClosure(direct, manifest)
+
+ const byName = new Map(manifest.packages.map((entry) => [entry.name, entry]))
+ const affectedPackages = affectedNames.map((name) => byName.get(name))
+ const suites = packageSets(affectedPackages)
+ const coverage = coverageSets(affectedPackages)
+ suites.integration = sorted([
+ ...suites.integration,
+ ...manifest.path_suites
+ .filter((entry) => paths.some((path) => pathWithin(path, normalizePath(entry.root))))
+ .flatMap((entry) => entry.integration_suites),
+ ])
+ if (full) {
+ suites.unit = sorted([...suites.unit, "scripts/ci"])
+ coverage.include_roots = sorted([...coverage.include_roots, "scripts/ci/**/*.mjs"])
+ }
+ const jobs = emptyJobs()
+ const sharedInputs = loadBlogSharedInputs()
+ let uiSpecs = []
+
+ if (full) {
+ uiSpecs = [manifest.ui.all]
+ } else if (blogContentOnly || blogPathsOnly) {
+ uiSpecs = [...manifest.ui.contracts.blog]
+ suites.integration = blogPathsOnly && !blogContentOnly ? [] : suites.integration
+ } else if (affectedPackages.some((entry) => entry.ui)) {
+ uiSpecs = [manifest.ui.all]
+ }
+
+ if (blogPathsOnly && !blogContentOnly) {
+ suites.integration = []
+ suites.windows = []
+ suites.linux = []
+ }
+
+ const blogBuild = full || paths.some((path) => isBlogBuildInput(path, sharedInputs))
+ const directWebRuntime = paths.some((path) => pathWithin(path, "src/web") && !BLOG_ROOT.test(path))
+ const lighthouse = full || (blogPathsOnly && !blogContentOnly) || directWebRuntime
+ const rust = full || affectedPackages.some((entry) => entry.rust)
+ const directPackages = direct.map((name) => byName.get(name))
+ const appArtifact = full || (!blogPathsOnly && [...affectedPackages, ...directPackages]
+ .some((entry) => entry.app_artifact))
+ const requiredChangedFiles = requiredCoverageFiles(changes, coverage.include_roots)
+
+ jobs.blog_build = blogBuild
+ jobs.static_checks = suites.static.length > 0
+ jobs.test_linux = suites.unit.length > 0 || suites.linux.length > 0
+ jobs.test_windows = suites.windows.length > 0
+ jobs.app_packed_artifact = appArtifact
+ jobs.e2e = suites.integration.length > 0
+ jobs.rust = rust
+ jobs.lighthouse = lighthouse
+ jobs.ui_e2e = uiSpecs.length > 0
+ jobs.merge_reports = jobs.ui_e2e
+
+ const basePlan = {
+ schema_version: manifest.schema_version,
+ policy_version: manifest.policy_version,
+ base_sha: options.baseSha || "",
+ head_sha: options.headSha || "",
+ diagnostic_only: options.diagnosticOnly === true,
+ changes,
paths,
+ change_class: planClass({ full, docsOnly, blogPathsOnly, direct }),
full,
- docs_only: effectiveDocsOnly,
- blog_only: effectiveBlogOnly,
- workflow_changed: workflowChanged,
- run_code_checks: runCodeChecks,
- run_windows: runCodeChecks && (full || app || cli || daemon || shared),
- run_e2e: runCodeChecks && (full || web || shared || cli || daemon || worker || integration),
- run_ui_e2e: !effectiveBlogOnly && !blogCodeOnly && runCodeChecks && (full || web || shared || wsDo),
- run_rust: runCodeChecks && (full || desktop),
- run_lighthouse: runCodeChecks && (full || web),
- run_knip: runCodeChecks && (full || app || cli || shared || web || worker),
- run_app_packed_artifact: full || appPackedArtifactInput,
- run_blog_build: full || paths.some(isBlogBuildInput),
+ full_reason: reason,
+ docs_only: docsOnly && !full,
+ blog_only: blogContentOnly && !full,
+ workflow_changed: paths.some((path) => WORKFLOW.test(path)),
+ packages: {
+ direct,
+ affected: affectedNames,
+ },
+ suites,
+ ui: { specs: sorted(uiSpecs) },
+ coverage: {
+ ...coverage,
+ required_changed_files: requiredChangedFiles,
+ },
+ jobs,
+ }
+ return { ...basePlan, plan_hash: hashPlan(basePlan) }
+}
+
+export function classifyPaths(inputPaths, options = {}) {
+ return buildExecutionPlan(
+ inputPaths.map((path) => ({ status: "M", path })),
+ options,
+ )
+}
+
+export function validateExecutionPlan(plan, options = {}) {
+ const manifest = validateScopeManifest(options.manifest || loadScopeManifest())
+ if (plan.schema_version !== manifest.schema_version || plan.policy_version !== manifest.policy_version) {
+ throw new Error("execution plan schema/policy version mismatch")
+ }
+ if (hashPlan(plan) !== plan.plan_hash) throw new Error("execution plan hash mismatch")
+
+ const packageNames = new Set(manifest.packages.map((entry) => entry.name))
+ for (const name of [...plan.packages.direct, ...plan.packages.affected, ...plan.suites.static, ...plan.suites.build, ...plan.suites.knip]) {
+ if (!packageNames.has(name)) throw new Error(`unknown execution-plan package: ${name}`)
+ }
+ for (const suite of plan.suites.windows) {
+ if (!manifest.suites.windows.includes(suite)) throw new Error(`unknown execution-plan windows suite: ${suite}`)
+ }
+ for (const suite of plan.suites.integration) {
+ if (!manifest.suites.integration.includes(suite)) throw new Error(`unknown execution-plan integration suite: ${suite}`)
+ }
+ for (const suite of plan.suites.linux) {
+ if (!manifest.suites.linux.includes(suite)) throw new Error(`unknown execution-plan linux suite: ${suite}`)
+ }
+ const inventory = discoverE2eSpecs(resolve(ROOT, manifest.ui.root))
+ for (const spec of plan.ui.specs) {
+ if (spec !== manifest.ui.all && !inventory.includes(spec)) throw new Error(`unknown execution-plan UI spec: ${spec}`)
+ }
+ return plan
+}
+
+export function projectPlan(plan) {
+ validateExecutionPlan(plan)
+ const uiSpecs = plan.ui.specs.includes("all") ? discoverE2eSpecs() : plan.ui.specs
+ const e2eMatrix = uiSpecs.length > 0 ? createE2eMatrix(uiSpecs) : { include: [] }
+ return {
+ execution_plan: stablePlanJson(plan),
+ plan_hash: plan.plan_hash,
+ plan_schema_version: String(plan.schema_version),
+ full: String(plan.full),
+ base_sha: plan.base_sha,
+ head_sha: plan.head_sha,
+ blog_only: String(plan.blog_only),
+ run_blog_build: String(plan.jobs.blog_build),
+ run_code_checks: String(plan.jobs.static_checks || plan.jobs.test_linux),
+ run_static_checks: String(plan.jobs.static_checks),
+ run_linux: String(plan.jobs.test_linux),
+ run_windows: String(plan.jobs.test_windows),
+ run_e2e: String(plan.jobs.e2e),
+ run_ui_e2e: String(plan.jobs.ui_e2e),
+ run_rust: String(plan.jobs.rust),
+ run_lighthouse: String(plan.jobs.lighthouse),
+ run_knip: String(plan.suites.knip.length > 0),
+ run_app_packed_artifact: String(plan.jobs.app_packed_artifact),
+ static_packages: JSON.stringify(plan.suites.static),
+ build_packages: JSON.stringify(plan.suites.build),
+ knip_packages: JSON.stringify(plan.suites.knip),
+ unit_test_roots: JSON.stringify(plan.suites.unit),
+ windows_suites: JSON.stringify(plan.suites.windows),
+ integration_suites: JSON.stringify(plan.suites.integration),
+ linux_suites: JSON.stringify(plan.suites.linux),
+ coverage_targets: JSON.stringify(plan.coverage.targets),
+ coverage_include_roots: JSON.stringify(plan.coverage.include_roots),
+ coverage_required_changed_files: JSON.stringify(plan.coverage.required_changed_files),
+ ui_specs: JSON.stringify(plan.ui.specs),
+ e2e_matrix: JSON.stringify(e2eMatrix),
}
}
export function parseNameStatus(buffer) {
const fields = buffer.toString("utf8").split("\0")
- const paths = []
- for (let index = 0; index < fields.length; ) {
+ const changes = []
+ for (let index = 0; index < fields.length;) {
const status = fields[index++]
if (!status) continue
const firstPath = fields[index++]
- if (firstPath) paths.push(firstPath)
+ if (!firstPath) throw new Error(`missing path for git status ${status}`)
if (status.startsWith("R") || status.startsWith("C")) {
const secondPath = fields[index++]
- if (secondPath) paths.push(secondPath)
+ if (!secondPath) throw new Error(`missing destination for git status ${status}`)
+ changes.push({ status, old_path: firstPath, path: secondPath })
+ } else {
+ changes.push({ status, path: firstPath })
}
}
- return paths
+ return normalizedChanges(changes)
}
-function readChangedPaths(args) {
+function readChangedFiles(args) {
if (args.forceFull) return []
- if (args.pathsFile) {
- return readFileSync(resolve(args.pathsFile), "utf8")
- .split(/\r?\n/)
- .filter(Boolean)
- }
- if (!args.base || !args.head) {
- throw new Error("Both --base and --head are required")
- }
+ if (!args.base || !args.head) throw new Error("Both --base and --head are required")
const diff = execFileSync(
"git",
- ["diff", "--name-status", "-z", "--find-renames", `${args.base}...${args.head}`],
- { stdio: ["ignore", "pipe", "pipe"] }
+ ["diff", "--name-status", "-z", "--find-renames", "--find-copies", `${args.base}...${args.head}`],
+ { cwd: ROOT, stdio: ["ignore", "pipe", "pipe"] },
)
return parseNameStatus(diff)
}
function parseArgs(argv) {
- const args = { forceFull: false }
+ const args = { forceFull: false, diagnosticOnly: false }
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index]
if (arg === "--force-full") {
args.forceFull = true
continue
}
+ if (arg === "--diagnostic-only") {
+ args.diagnosticOnly = true
+ continue
+ }
const key = arg.replace(/^--/, "").replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
args[key] = argv[++index]
}
return args
}
-function writeOutputs(path, result) {
- const output = Object.entries(result)
- .filter(([key]) => key !== "paths")
- .map(([key, value]) => `${key}=${String(value)}`)
- .join("\n")
- appendFileSync(path, `${output}\n`)
+function writeOutputs(path, projection) {
+ appendFileSync(path, `${Object.entries(projection).map(([key, value]) => `${key}=${value}`).join("\n")}\n`)
}
-function writeSummary(path, result, fallbackReason) {
- const flags = Object.entries(result)
- .filter(([key]) => key !== "paths")
+function writeSummary(path, plan, fallbackReason) {
+ const projection = projectPlan(plan)
+ const flags = Object.entries(projection)
+ .filter(([key]) => key.startsWith("run_") || key === "full" || key === "plan_hash")
.map(([key, value]) => `| \`${key}\` | \`${value}\` |`)
.join("\n")
- const changed = result.paths.map((item) => `- \`${item}\``).join("\n") || "- none"
+ const changed = plan.changes.map((change) => (
+ `- \`${change.status}\` ${change.old_path ? `\`${change.old_path}\` → ` : ""}\`${change.path}\``
+ )).join("\n") || "- none"
const fallback = fallbackReason ? `\nFail-closed reason: ${fallbackReason}\n` : ""
appendFileSync(
path,
- `## CI scope\n${fallback}\n| Output | Value |\n| --- | --- |\n${flags}\n\n<details><summary>Changed paths</summary>\n\n${changed}\n\n</details>\n`
+ `## CI execution plan\n${fallback}\nSchema/policy: \`${plan.schema_version}/${plan.policy_version}\` \nPlan hash: \`${plan.plan_hash}\` \nClass: \`${plan.change_class}\` \nDiagnostic only: \`${plan.diagnostic_only}\`\n\n| Projection | Value |\n| --- | --- |\n${flags}\n\n<details><summary>Changed paths</summary>\n\n${changed}\n\n</details>\n`,
)
}
export function runCli(argv) {
const args = parseArgs(argv)
- let result
+ let plan
let fallbackReason = ""
try {
- result = classifyPaths(readChangedPaths(args), { forceFull: args.forceFull })
+ plan = buildExecutionPlan(readChangedFiles(args), {
+ baseSha: args.base || "",
+ headSha: args.head || "",
+ forceFull: args.forceFull,
+ diagnosticOnly: args.diagnosticOnly,
+ })
} catch (error) {
fallbackReason = error instanceof Error ? error.message : String(error)
- result = classifyPaths([], { forceFull: true })
+ plan = buildExecutionPlan([], {
+ baseSha: args.base || "",
+ headSha: args.head || "",
+ forceFull: true,
+ fallbackReason,
+ diagnosticOnly: args.diagnosticOnly,
+ })
}
- if (args.output) writeOutputs(args.output, result)
- if (args.summary) writeSummary(args.summary, result, fallbackReason)
- if (!args.output) process.stdout.write(`${JSON.stringify(result, null, 2)}\n`)
+ validateExecutionPlan(plan)
+ const projection = projectPlan(plan)
+ if (args.output) writeOutputs(args.output, projection)
+ if (args.planFile) writeFileSync(resolve(args.planFile), `${stablePlanJson(plan)}\n`)
+ if (args.summary) writeSummary(args.summary, plan, fallbackReason)
+ if (!args.output) process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`)
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
diff --git a/scripts/ci/changed-scopes.test.ts b/scripts/ci/changed-scopes.test.ts
index 7873b0653..d15647957 100644
--- a/scripts/ci/changed-scopes.test.ts
+++ b/scripts/ci/changed-scopes.test.ts
@@ -1,182 +1,576 @@
-import { mkdtempSync, readFileSync, rmSync } from "node:fs"
+import { execFileSync } from "node:child_process"
+import { createHash } from "node:crypto"
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
import { tmpdir } from "node:os"
import { join } from "node:path"
-import { describe, expect, it } from "vitest"
-import { classifyPaths, parseNameStatus, runCli } from "./changed-scopes.mjs"
+import { describe, expect, it, vi } from "vitest"
+import {
+ buildExecutionPlan,
+ classifyPaths,
+ loadScopeManifest,
+ parseNameStatus,
+ projectPlan,
+ runCli,
+ stablePlanJson,
+ validateExecutionPlan,
+ validateScopeManifest,
+} from "./changed-scopes.mjs"
-describe("classifyPaths", () => {
- it("selects the strict blog fast path for MDX and blog assets", () => {
- const result = classifyPaths([
+const sha = (character: string) => character.repeat(40)
+const modified = (...paths: string[]) => paths.map((path) => ({ status: "M", path }))
+
+function plan(paths: string[], options = {}) {
+ return buildExecutionPlan(modified(...paths), {
+ baseSha: sha("a"),
+ headSha: sha("b"),
+ ...options,
+ })
+}
+
+function resignPlan(value) {
+ const { plan_hash: _ignored, ...unsigned } = value
+ return {
+ ...value,
+ plan_hash: createHash("sha256").update(stablePlanJson(unsigned)).digest("hex"),
+ }
+}
+
+function workspaceFixture({
+ codecov = (source: string) => source,
+} = {}) {
+ const manifest = loadScopeManifest()
+ const root = mkdtempSync(join(tmpdir(), "alook-ci-workspace-"))
+ writeFileSync(
+ join(root, "pnpm-workspace.yaml"),
+ readFileSync("pnpm-workspace.yaml", "utf8"),
+ )
+ writeFileSync(
+ join(root, "codecov.yml"),
+ codecov(readFileSync("codecov.yml", "utf8")),
+ )
+ for (const scopePackage of manifest.packages) {
+ const packageRoot = join(root, scopePackage.root)
+ mkdirSync(packageRoot, { recursive: true })
+ writeFileSync(
+ join(packageRoot, "package.json"),
+ readFileSync(join(scopePackage.root, "package.json"), "utf8"),
+ )
+ }
+ return { manifest, root }
+}
+
+function localGitEnvironmentNames() {
+ return execFileSync("git", ["rev-parse", "--local-env-vars"], { encoding: "utf8" })
+ .split(/\r?\n/)
+ .filter(Boolean)
+}
+
+function cleanGitEnvironment() {
+ const environment = { ...process.env }
+ for (const name of localGitEnvironmentNames()) delete environment[name]
+ return environment
+}
+
+function gitDiffFixture() {
+ const root = mkdtempSync(join(tmpdir(), "alook-ci-git-"))
+ const runGit = (...args: string[]) => execFileSync("git", args, {
+ cwd: root,
+ env: cleanGitEnvironment(),
+ stdio: "ignore",
+ })
+ const commit = (message: string) => runGit(
+ "-c", "user.name=Alook CI",
+ "-c", "commit.gpgsign=false",
+ "commit", "--quiet", "-m", message,
+ )
+
+ runGit("init", "--quiet")
+ writeFileSync(join(root, "README.md"), "baseline\n")
+ runGit("add", "README.md")
+ commit("baseline")
+
+ const changedPath = "src/cli/src/commands/inbox.ts"
+ mkdirSync(join(root, "src/cli/src/commands"), { recursive: true })
+ writeFileSync(join(root, changedPath), "export const fixture = true\n")
+ runGit("add", changedPath)
+ commit("change cli")
+
+ return { changedPath, root }
+}
+
+describe("canonical execution plan", () => {
+ it("routes Blog runtime through the complete Web contract and only UI spec 54", () => {
+ const result = plan(["src/web/blog/src/app/page.tsx"])
+
+ expect(result.change_class).toBe("blog")
+ expect(result.packages.direct).toEqual(["@alook/web"])
+ expect(result.packages.affected).toEqual(["@alook/web"])
+ expect(result.suites.unit).toEqual(["src/web"])
+ expect(result.coverage.targets).toEqual(["web"])
+ expect(result.coverage.include_roots).toEqual(["src/web/**/*.{ts,tsx,js,jsx}"])
+ expect(result.ui.specs).toEqual(["54-blog-multizone.spec.ts"])
+ expect(result.suites.integration).toEqual([])
+ expect(result.suites.windows).toEqual([])
+ expect(result.jobs.blog_build).toBe(true)
+ expect(result.jobs.lighthouse).toBe(true)
+ expect(result.jobs.app_packed_artifact).toBe(false)
+ })
+
+ it("keeps Blog content on build and its cross-worker contract without package coverage", () => {
+ const result = plan([
"src/web/blog/src/content/example.mdx",
"src/web/blog/public/blog/example/hero.webp",
])
+ expect(result.change_class).toBe("blog")
expect(result.blog_only).toBe(true)
- expect(result.run_code_checks).toBe(false)
- expect(result.run_ui_e2e).toBe(false)
- expect(result.run_blog_build).toBe(true)
+ expect(result.packages.direct).toEqual(["@alook/web"])
+ expect(result.packages.affected).toEqual([])
+ expect(result.suites.unit).toEqual([])
+ expect(result.coverage.include_roots).toEqual([])
+ expect(result.ui.specs).toEqual(["54-blog-multizone.spec.ts"])
+ expect(result.jobs.blog_build).toBe(true)
})
- it("selects no package work for markdown-only changes", () => {
- const result = classifyPaths(["README.md", "docs/operations.md"])
+ it("selects only the CLI package and its explicit platform suites", () => {
+ const result = plan(["src/cli/src/commands/inbox.ts"])
- expect(result.docs_only).toBe(true)
- expect(result.run_code_checks).toBe(false)
+ expect(result.change_class).toBe("package")
+ expect(result.packages.affected).toEqual(["@alook/cli"])
+ expect(result.suites.static).toEqual(["@alook/cli"])
+ expect(result.suites.unit).toEqual(["src/cli"])
+ expect(result.suites.integration).toEqual(["cli"])
+ expect(result.suites.windows).toEqual(["cli"])
+ expect(result.ui.specs).toEqual([])
+ expect(result.coverage.targets).toEqual(["cli"])
+ expect(result.jobs.app_packed_artifact).toBe(false)
})
- it("disables the blog fast path when content is mixed with docs", () => {
- const result = classifyPaths([
- "src/web/blog/src/content/example.mdx",
- "README.md",
- ])
+ it("routes integration-owned paths without widening to unrelated suites", () => {
+ const cli = plan(["tests/integration/cli/session-resume.test.ts"])
+ expect(cli.suites.integration).toEqual(["cli"])
+ expect(cli.jobs.e2e).toBe(true)
+ expect(cli.jobs.static_checks).toBe(false)
- expect(result.blog_only).toBe(false)
- expect(result.full).toBe(true)
- })
+ const daemon = plan(["tests/integration/daemon/lifecycle.test.ts"])
+ expect(daemon.suites.integration).toEqual(["daemon"])
+
+ const undeclared = plan(["tests/integration/future/case.test.ts"])
+ expect(undeclared.full).toBe(true)
+ expect(undeclared.full_reason).toBe("unknown_path")
+ expect(undeclared.suites.integration).toEqual(["cli", "daemon", "web"])
- it("keeps Windows for app, shared, and CLI changes", () => {
- const app = classifyPaths(["src/app/src/lib/install.ts"])
- expect(app.full).toBe(false)
- expect(app.run_windows).toBe(true)
- expect(classifyPaths(["src/shared/src/schema.ts"]).run_windows).toBe(true)
- expect(classifyPaths(["src/cli/src/index.ts"]).run_windows).toBe(true)
- expect(classifyPaths(["src/web/src/app/page.tsx"]).run_windows).toBe(false)
+ const trailingSlashManifest = structuredClone(loadScopeManifest())
+ const cliPathSuite = trailingSlashManifest.path_suites.find((entry) => (
+ entry.root === "tests/integration/cli"
+ ))
+ expect(cliPathSuite).toBeDefined()
+ cliPathSuite!.root += "/"
+ const canonicalized = buildExecutionPlan(
+ modified("tests/integration/cli/session-resume.test.ts"),
+ {
+ baseSha: sha("a"),
+ headSha: sha("b"),
+ manifest: trailingSlashManifest,
+ },
+ )
+ expect(canonicalized.full).toBe(false)
+ expect(canonicalized.suites.integration).toEqual(["cli"])
})
- it("routes web runtime changes through browser and Lighthouse checks", () => {
- const result = classifyPaths(["src/web/src/app/page.tsx"])
+ it("expands shared changes through the complete workspace dependent closure", () => {
+ const result = plan(["src/shared/src/schema.ts"])
- expect(result.run_ui_e2e).toBe(true)
- expect(result.run_lighthouse).toBe(true)
- expect(result.run_e2e).toBe(true)
- expect(result.run_app_packed_artifact).toBe(true)
+ expect(result.change_class).toBe("shared")
+ expect(result.packages.affected).toEqual([
+ "@alook/app",
+ "@alook/cli",
+ "@alook/daemon",
+ "@alook/email-worker",
+ "@alook/shared",
+ "@alook/test-utils",
+ "@alook/wake-worker",
+ "@alook/web",
+ "@alook/ws-do",
+ ])
+ expect(result.suites.integration).toEqual(["cli", "daemon", "web"])
+ expect(result.suites.windows).toEqual(["app", "cli", "daemon", "shared"])
+ expect(result.ui.specs).toEqual(["all"])
+ expect(result.jobs.app_packed_artifact).toBe(true)
+ expect(result.jobs.rust).toBe(false)
})
- it("builds Blog for nested code and canonical shared inputs", () => {
- for (const path of [
- "src/web/blog/custom-worker.ts",
- "src/web/blog/src/lib/blog/posts/index.ts",
- "src/web/src/components/public-layout.tsx",
- "src/web/src/lib/seo/site-metadata.ts",
+ it("fails closed for CI policy, root config, unknown paths, empty diffs, and force full", () => {
+ for (const paths of [
+ [".github/workflows/ci.yml"],
+ ["pnpm-lock.yaml"],
+ ["docs/architecture.png"],
+ [".claude/settings.json"],
+ ["src/future-package/src/index.ts"],
+ ["unexpected/file.txt"],
+ [],
]) {
- expect(classifyPaths([path]).run_blog_build, path).toBe(true)
+ const result = plan(paths)
+ expect(result.full, paths.join(",") || "empty").toBe(true)
+ expect(result.ui.specs).toEqual(["all"])
+ expect(result.suites.integration).toEqual(["cli", "daemon", "web"])
+ expect(result.suites.windows).toEqual(["agent-driver", "app", "cli", "daemon", "shared"])
}
- const blogCode = classifyPaths(["src/web/blog/custom-worker.ts"])
- expect(blogCode.run_code_checks).toBe(true)
- expect(blogCode.run_ui_e2e).toBe(false)
- expect(blogCode.run_app_packed_artifact).toBe(false)
- expect(classifyPaths(["src/web/src/app/api/health/route.ts"]).run_blog_build).toBe(false)
+ expect(plan(["README.md"], { forceFull: true }).full).toBe(true)
+ expect(plan([".github/workflows/ci.yml"]).full_reason).toBe("policy_change")
})
- it("fails closed for every app bundle input while excluding stripped blog-only inputs", () => {
- for (const path of [
- "src/app/src/index.ts",
- "src/app/README.md",
- "src/daemon/src/index.ts",
- "src/daemon/agent-driver/src/index.ts",
- "src/daemon/README.md",
- "src/web/custom-worker.ts",
- "src/web/src/lib/worker-runtime.ts",
- "src/web/wrangler.toml",
- "src/web/migrations/0001.sql",
- "src/web/README.md",
- "src/shared/src/index.ts",
- "src/shared/README.md",
- "src/email-worker/src/index.ts",
- "src/email-worker/README.md",
- "src/ws-do/src/index.ts",
- "src/ws-do/README.md",
- "src/wake-worker/src/index.ts",
- "src/wake-worker/README.md",
- "pnpm-lock.yaml",
- ".github/workflows/publish-app.yml",
+ it("keeps Markdown-only changes empty and mixed content fail-closed", () => {
+ const docs = plan(["README.md", "docs/operations.md"])
+ expect(docs.docs_only).toBe(true)
+ expect(Object.values(docs.jobs).some(Boolean)).toBe(false)
+
+ const mixed = plan(["README.md", "src/web/blog/src/content/example.mdx"])
+ expect(mixed.full).toBe(true)
+ expect(mixed.full_reason).toBe("mixed_content")
+ })
+
+ it("preserves the packed app contract for package-local Markdown", () => {
+ expect(plan(["src/app/README.md"]).jobs.app_packed_artifact).toBe(true)
+ expect(plan(["src/daemon/README.md"]).jobs.app_packed_artifact).toBe(true)
+ expect(plan(["src/shared/README.md"]).jobs.app_packed_artifact).toBe(true)
+ expect(plan(["src/cli/README.md"]).jobs.app_packed_artifact).toBe(false)
+ })
+
+ it("takes a monotonic union for multiple paths", () => {
+ const cli = plan(["src/cli/src/index.ts"])
+ const web = plan(["src/web/src/app/page.tsx"])
+ const combined = plan(["src/web/src/app/page.tsx", "src/cli/src/index.ts"])
+
+ for (const value of cli.packages.affected) expect(combined.packages.affected).toContain(value)
+ for (const value of web.packages.affected) expect(combined.packages.affected).toContain(value)
+ for (const value of cli.suites.integration) expect(combined.suites.integration).toContain(value)
+ for (const value of web.suites.integration) expect(combined.suites.integration).toContain(value)
+ })
+
+ it("produces byte-identical plans, hashes, and mechanical projections", () => {
+ const first = plan(["src/cli/src/index.ts", "src/web/src/app/page.tsx"])
+ const second = plan(["src/web/src/app/page.tsx", "src/cli/src/index.ts"])
+
+ expect(first).toEqual(second)
+ expect(stablePlanJson(first)).toBe(stablePlanJson(second))
+ expect(validateExecutionPlan(first)).toEqual(first)
+ const projection = projectPlan(first)
+ expect(JSON.parse(projection.execution_plan)).toEqual(first)
+ expect(projection.plan_hash).toBe(first.plan_hash)
+ expect(JSON.parse(projection.static_packages)).toEqual(first.suites.static)
+ expect(JSON.parse(projection.integration_suites)).toEqual(first.suites.integration)
+
+ expect(() => validateExecutionPlan({ ...first, head_sha: sha("c") })).toThrow("hash")
+ })
+
+ it("rejects every stale identifier in an otherwise authentic signed plan", () => {
+ const current = plan(["src/shared/src/schema.ts"])
+ expect(() => validateExecutionPlan({ ...current, schema_version: 2 }))
+ .toThrow("schema/policy")
+
+ for (const [section, key, expected] of [
+ ["packages", "direct", "package"],
+ ["suites", "windows", "windows suite"],
+ ["suites", "integration", "integration suite"],
+ ["suites", "linux", "linux suite"],
]) {
- expect(classifyPaths([path]).run_app_packed_artifact, path).toBe(true)
+ const altered = structuredClone(current)
+ altered[section][key] = ["future"]
+ expect(() => validateExecutionPlan(resignPlan(altered))).toThrow(expected)
}
- expect(classifyPaths(["src/web/blog/src/content/example.mdx"]).run_app_packed_artifact).toBe(false)
- expect(classifyPaths(["src/web/blog/public/blog/example/hero.webp"]).run_app_packed_artifact).toBe(false)
- expect(classifyPaths(["src/web/blog/custom-worker.ts"]).run_app_packed_artifact).toBe(false)
- expect(classifyPaths(["README.md"]).run_app_packed_artifact).toBe(false)
- expect(classifyPaths(["src/app/README.md"]).run_code_checks).toBe(false)
+
+ const staleUi = structuredClone(current)
+ staleUi.ui.specs = ["future.spec.ts"]
+ expect(() => validateExecutionPlan(resignPlan(staleUi))).toThrow("UI spec")
})
+})
- it("routes CLI and daemon integration test changes through Linux E2E", () => {
+describe("coverage name-status contract", () => {
+ it("requires surviving A/M and rename/copy new sides but not deleted or old paths", () => {
+ const result = buildExecutionPlan([
+ { status: "A", path: "src/cli/src/added.ts" },
+ { status: "M", path: "src/cli/src/modified.ts" },
+ { status: "D", path: "src/cli/src/deleted.ts" },
+ { status: "R100", old_path: "src/cli/src/old.ts", path: "src/cli/src/renamed.ts" },
+ { status: "C090", old_path: "src/cli/src/source.ts", path: "src/cli/src/copied.ts" },
+ ], { baseSha: sha("a"), headSha: sha("b") })
+
+ expect(result.coverage.required_changed_files).toEqual([
+ "src/cli/src/added.ts",
+ "src/cli/src/copied.ts",
+ "src/cli/src/modified.ts",
+ "src/cli/src/renamed.ts",
+ ])
+ })
+
+ it("requires only files selected by the actual coverage globs", () => {
for (const path of [
- "tests/integration/cli/session-resume.test.ts",
- "tests/integration/daemon/control-plane.test.ts",
+ "src/app/scripts/app-packed-artifact.mjs",
+ "src/cli/scripts/prepare-dist.mjs",
+ "src/web/vitest.runtime.config.mts",
]) {
- const result = classifyPaths([path])
- expect(result.full).toBe(false)
- expect(result.run_e2e).toBe(true)
+ expect(plan([path]).coverage.required_changed_files).toEqual([])
}
+
+ expect(plan(["scripts/ci/changed-scopes.mjs"]).coverage.required_changed_files)
+ .toEqual(["scripts/ci/changed-scopes.mjs"])
})
- it("only accepts top-level MDX posts for the blog fast path", () => {
- const result = classifyPaths(["src/web/blog/src/content/nested/example.mdx"])
+ it("parses NUL-delimited statuses without losing rename/copy identity", () => {
+ const input = Buffer.from(
+ "M\0src/cli/src/a.ts\0D\0src/cli/src/deleted.ts\0R100\0src/cli/src/old.ts\0src/cli/src/new.ts\0C090\0src/shared/src/a.ts\0src/shared/src/b.ts\0",
+ )
- expect(result.blog_only).toBe(false)
- expect(result.run_code_checks).toBe(true)
- expect(result.run_ui_e2e).toBe(true)
- expect(result.run_lighthouse).toBe(true)
+ expect(parseNameStatus(input)).toEqual([
+ { status: "M", path: "src/cli/src/a.ts" },
+ { status: "D", path: "src/cli/src/deleted.ts" },
+ { status: "R100", old_path: "src/cli/src/old.ts", path: "src/cli/src/new.ts" },
+ { status: "C090", old_path: "src/shared/src/a.ts", path: "src/shared/src/b.ts" },
+ ])
})
- it("routes desktop-only changes through Rust without UI E2E", () => {
- const result = classifyPaths(["src/desktop/src-tauri/src/lib.rs"])
+ it("rejects truncated NUL-delimited status records", () => {
+ expect(() => parseNameStatus(Buffer.from("M\0"))).toThrow("missing path")
+ expect(() => parseNameStatus(Buffer.from("R100\0old.ts\0"))).toThrow("missing destination")
+ })
+
+ it("sorts equal destination paths by old path and then status", () => {
+ const input = Buffer.from([
+ "R100", "src/cli/src/z.ts", "src/cli/src/same.ts",
+ "R090", "src/cli/src/a.ts", "src/cli/src/same.ts",
+ "C100", "src/cli/src/a.ts", "src/cli/src/same.ts",
+ "M", "src/cli/src/same.ts",
+ "A", "src/cli/src/same.ts",
+ "",
+ ].join("\0"))
- expect(result.run_rust).toBe(true)
- expect(result.run_ui_e2e).toBe(false)
+ expect(parseNameStatus(input)).toEqual([
+ { status: "A", path: "src/cli/src/same.ts" },
+ { status: "M", path: "src/cli/src/same.ts" },
+ { status: "C100", old_path: "src/cli/src/a.ts", path: "src/cli/src/same.ts" },
+ { status: "R090", old_path: "src/cli/src/a.ts", path: "src/cli/src/same.ts" },
+ { status: "R100", old_path: "src/cli/src/z.ts", path: "src/cli/src/same.ts" },
+ ])
})
+})
- it("fails closed for workflows, root configuration, unknown paths, and empty diffs", () => {
- expect(classifyPaths([".github/workflows/ci.yml"]).full).toBe(true)
- expect(classifyPaths([".github/dependabot.yml"]).full).toBe(true)
- expect(classifyPaths(["scripts/bump-version.mjs"]).full).toBe(true)
- expect(classifyPaths(["pnpm-lock.yaml"]).full).toBe(true)
- expect(classifyPaths(["src/future-package/src/index.ts"]).full).toBe(true)
- expect(classifyPaths(["unexpected/file.txt"]).full).toBe(true)
- expect(classifyPaths([]).full).toBe(true)
+describe("scope manifest", () => {
+ it("matches workspace package manifests and the complete UI inventory", () => {
+ const manifest = loadScopeManifest()
+ expect(() => validateScopeManifest(manifest)).not.toThrow()
+ expect(manifest.schema_version).toBe(1)
+ expect(manifest.ui.contracts.blog).toEqual(["54-blog-multizone.spec.ts"])
})
- it("honors an explicit full run", () => {
- const result = classifyPaths(["src/web/blog/src/content/example.mdx"], {
- forceFull: true,
- })
+ it("rejects invalid versions, registries, packages, paths, targets, and specs", () => {
+ const manifest = loadScopeManifest()
- expect(result.full).toBe(true)
- expect(result.blog_only).toBe(false)
- expect(result.run_windows).toBe(true)
- expect(result.run_app_packed_artifact).toBe(true)
+ expect(() => validateScopeManifest({ ...manifest, schema_version: 2 }))
+ .toThrow("schema/policy")
+ expect(() => validateScopeManifest({ ...manifest, packages: [] }))
+ .toThrow("packages are required")
+
+ const duplicateName = structuredClone(manifest)
+ duplicateName.packages[1].name = duplicateName.packages[0].name
+ expect(() => validateScopeManifest(duplicateName)).toThrow("names must be unique")
+
+ const brokenSuite = structuredClone(manifest)
+ brokenSuite.packages[0].windows_suites = ["future"]
+ expect(() => validateScopeManifest(brokenSuite)).toThrow("windows suite")
+
+ const duplicateSuite = structuredClone(manifest)
+ duplicateSuite.suites.windows.push(duplicateSuite.suites.windows[0])
+ expect(() => validateScopeManifest(duplicateSuite)).toThrow("IDs must be unique")
+
+ const registryDrift = structuredClone(manifest)
+ registryDrift.suites.windows.push("future")
+ expect(() => validateScopeManifest(registryDrift)).toThrow("registry")
+
+ const duplicateRoot = structuredClone(manifest)
+ duplicateRoot.packages[1].root = duplicateRoot.packages[0].root
+ expect(() => validateScopeManifest(duplicateRoot)).toThrow("root")
+
+ const duplicatePathRoot = structuredClone(manifest)
+ duplicatePathRoot.path_suites.push(structuredClone(duplicatePathRoot.path_suites[0]))
+ expect(() => validateScopeManifest(duplicatePathRoot)).toThrow("path suite roots")
+
+ const unknownPathSuite = structuredClone(manifest)
+ unknownPathSuite.path_suites[0].integration_suites = ["future"]
+ expect(() => validateScopeManifest(unknownPathSuite)).toThrow("path integration suite")
+
+ const missingPackage = structuredClone(manifest)
+ missingPackage.packages.pop()
+ expect(() => validateScopeManifest(missingPackage)).toThrow("pnpm-workspace")
+
+ const packageMismatch = structuredClone(manifest)
+ packageMismatch.packages[0].name = "@alook/future"
+ expect(() => validateScopeManifest(packageMismatch)).toThrow("package manifest mismatch")
+
+ const unknownCodecov = structuredClone(manifest)
+ const targetPackage = unknownCodecov.packages.find((entry) => entry.codecov_target)
+ expect(targetPackage).toBeDefined()
+ targetPackage!.codecov_target = "future"
+ expect(() => validateScopeManifest(unknownCodecov)).toThrow("unknown Codecov target")
+
+ const codecovDrift = structuredClone(manifest)
+ codecovDrift.codecov_targets.shared.target = 51
+ expect(() => validateScopeManifest(codecovDrift)).toThrow("Codecov target")
+
+ expect(() => validateScopeManifest(manifest, { specs: [] })).toThrow("unknown UI spec")
})
-})
-describe("parseNameStatus", () => {
- it("includes both sides of renames and copies", () => {
- const input = Buffer.from(
- "M\0src/web/blog/src/content/a.mdx\0R100\0src/cli/a.ts\0src/web/blog/src/content/a.mdx\0C090\0old.ts\0new.ts\0"
- )
+ it("validates the same Codecov contracts from a CRLF checkout", () => {
+ const { manifest, root } = workspaceFixture({
+ codecov: (source) => source.replaceAll(/\r?\n/g, "\r\n"),
+ })
+ try {
+ expect(() => validateScopeManifest(manifest, {
+ root,
+ specs: manifest.ui.contracts.blog,
+ })).not.toThrow()
+ } finally {
+ rmSync(root, { recursive: true, force: true })
+ }
+ })
- expect(parseNameStatus(input)).toEqual([
- "src/web/blog/src/content/a.mdx",
- "src/cli/a.ts",
- "src/web/blog/src/content/a.mdx",
- "old.ts",
- "new.ts",
- ])
+ it("rejects malformed Codecov sections and package test contracts", () => {
+ for (const codecov of [
+ (source: string) => source.replace(" project:", " projects:"),
+ (source: string) => source.replace(/^ ([^:\n]+):$/m, " future:"),
+ ]) {
+ const { manifest, root } = workspaceFixture({ codecov })
+ try {
+ expect(() => validateScopeManifest(manifest, {
+ root,
+ specs: manifest.ui.contracts.blog,
+ })).toThrow("codecov.yml")
+ } finally {
+ rmSync(root, { recursive: true, force: true })
+ }
+ }
+
+ const { manifest, root } = workspaceFixture()
+ try {
+ const unitPackage = manifest.packages.find((entry) => entry.unit_root)
+ expect(unitPackage).toBeDefined()
+ const packagePath = join(root, unitPackage!.root, "package.json")
+ const packageManifest = JSON.parse(readFileSync(packagePath, "utf8"))
+ delete packageManifest.scripts.test
+ writeFileSync(packagePath, JSON.stringify(packageManifest))
+ expect(() => validateScopeManifest(manifest, {
+ root,
+ specs: manifest.ui.contracts.blog,
+ })).toThrow("declares test scope")
+ } finally {
+ rmSync(root, { recursive: true, force: true })
+ }
})
})
-describe("runCli", () => {
- it("writes full-CI outputs when the diff cannot be resolved", () => {
+describe("compatibility and CLI fail-closed behavior", () => {
+ it("keeps classifyPaths as a canonical-plan adapter", () => {
+ const result = classifyPaths(["src/desktop/src-tauri/src/lib.rs"])
+ expect(result.jobs.rust).toBe(true)
+ expect(result.ui.specs).toEqual([])
+ })
+
+ it("writes a full plan when the diff cannot be resolved", () => {
const directory = mkdtempSync(join(tmpdir(), "alook-ci-scope-"))
const output = join(directory, "output")
+ const planFile = join(directory, "plan.json")
+ const summary = join(directory, "summary.md")
try {
- runCli(["--base", "missing-base", "--head", "missing-head", "--output", output])
+ runCli([
+ "--base", "missing-base",
+ "--head", "missing-head",
+ "--output", output,
+ "--plan-file", planFile,
+ "--summary", summary,
+ ])
const values = readFileSync(output, "utf8")
+ const writtenPlan = JSON.parse(readFileSync(planFile, "utf8"))
expect(values).toContain("full=true")
- expect(values).toContain("run_windows=true")
expect(values).toContain("run_ui_e2e=true")
- expect(values).toContain("run_app_packed_artifact=true")
+ expect(writtenPlan.full).toBe(true)
+ expect(writtenPlan.full_reason).toBe("classifier_error")
+ expect(readFileSync(summary, "utf8")).toContain("Fail-closed reason:")
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+
+ it("writes every canonical artifact for an explicit diagnostic full run", () => {
+ const directory = mkdtempSync(join(tmpdir(), "alook-ci-scope-"))
+ const output = join(directory, "output")
+ const planFile = join(directory, "plan.json")
+ const summary = join(directory, "summary.md")
+ try {
+ runCli([
+ "--force-full",
+ "--diagnostic-only",
+ "--output", output,
+ "--plan-file", planFile,
+ "--summary", summary,
+ ])
+
+ const writtenPlan = JSON.parse(readFileSync(planFile, "utf8"))
+ expect(writtenPlan).toMatchObject({
+ full: true,
+ full_reason: "forced",
+ diagnostic_only: true,
+ changes: [],
+ })
+ expect(readFileSync(output, "utf8")).toContain(`plan_hash=${writtenPlan.plan_hash}`)
+ expect(readFileSync(summary, "utf8")).toContain("- none")
+ expect(readFileSync(summary, "utf8")).not.toContain("Fail-closed reason:")
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+
+ it("reads a real Git diff and prints its canonical plan when no output file is requested", () => {
+ const fixture = gitDiffFixture()
+ const summary = join(fixture.root, "summary.md")
+ const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true)
+ const localEnvironment = localGitEnvironmentNames()
+ const previousEnvironment = new Map(
+ localEnvironment.map((name) => [name, process.env[name]]),
+ )
+ try {
+ for (const name of localEnvironment) delete process.env[name]
+ process.env.GIT_DIR = join(fixture.root, ".git")
+ process.env.GIT_WORK_TREE = fixture.root
+ runCli(["--base", "HEAD^", "--head", "HEAD", "--summary", summary])
+
+ const written = stdout.mock.calls.map(([value]) => String(value)).join("")
+ const writtenPlan = JSON.parse(written)
+ expect(writtenPlan.base_sha).toBe("HEAD^")
+ expect(writtenPlan.head_sha).toBe("HEAD")
+ expect(writtenPlan.changes).toEqual([{ status: "A", path: fixture.changedPath }])
+ expect(readFileSync(summary, "utf8")).toContain(fixture.changedPath)
+ } finally {
+ for (const name of localEnvironment) delete process.env[name]
+ for (const [name, value] of previousEnvironment) {
+ if (value !== undefined) process.env[name] = value
+ }
+ stdout.mockRestore()
+ rmSync(fixture.root, { recursive: true, force: true })
+ }
+ })
+
+ it("fails closed when the CLI SHA boundary is omitted", () => {
+ const directory = mkdtempSync(join(tmpdir(), "alook-ci-scope-"))
+ const output = join(directory, "output")
+ const planFile = join(directory, "plan.json")
+ try {
+ runCli(["--output", output, "--plan-file", planFile])
+ expect(JSON.parse(readFileSync(planFile, "utf8"))).toMatchObject({
+ full: true,
+ full_reason: "classifier_error",
+ })
} finally {
rmSync(directory, { recursive: true, force: true })
}
diff --git a/scripts/ci/characterization-guard.mjs b/scripts/ci/characterization-guard.mjs
new file mode 100644
index 000000000..cd7990d74
--- /dev/null
+++ b/scripts/ci/characterization-guard.mjs
@@ -0,0 +1,208 @@
+import { execFileSync, spawnSync } from "node:child_process"
+import { appendFileSync, readFileSync } from "node:fs"
+import { resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+import {
+ buildExecutionPlan,
+ loadScopeManifest,
+ parseNameStatus,
+ validateExecutionPlan,
+} from "./changed-scopes.mjs"
+
+const ROOT = resolve(fileURLToPath(new URL("../..", import.meta.url)))
+const FULL_SHA = /^[0-9a-f]{40}$/
+const CLASSES = new Set(["blog", "package", "shared"])
+
+export function validateCharacterization(input) {
+ const {
+ repository,
+ refName,
+ eventSha,
+ candidateSha,
+ fixtureSha,
+ expectedClass,
+ candidatePr,
+ refSha,
+ parentShas,
+ isAncestor,
+ fileMode,
+ changes,
+ plan,
+ integrityBlobs,
+ manifest,
+ } = input
+
+ for (const [name, value] of Object.entries({ eventSha, candidateSha, fixtureSha, refSha })) {
+ if (!FULL_SHA.test(value)) throw new Error(`${name} must be a full lowercase SHA`)
+ }
+ if (!CLASSES.has(expectedClass)) throw new Error("invalid expected characterization class")
+ if (refName !== `ci-characterization/${candidateSha.slice(0, 12)}/${expectedClass}`) {
+ throw new Error("characterization ref does not match the fixed branch prefix")
+ }
+ if (eventSha !== fixtureSha || refSha !== fixtureSha) {
+ throw new Error("fixture SHA must match both the dispatch event SHA and branch ref tip")
+ }
+ if (candidatePr?.state !== "open") throw new Error("candidate PR must remain open")
+ if (candidatePr?.head?.repo?.full_name !== repository) {
+ throw new Error("candidate PR and fixture must belong to the same repository")
+ }
+ if (candidatePr?.head?.sha !== candidateSha) {
+ throw new Error("candidate PR head no longer matches the locked candidate SHA")
+ }
+ if (parentShas.length !== 1 || parentShas[0] !== candidateSha) {
+ throw new Error("fixture must be a single-parent direct child of the candidate")
+ }
+ if (!isAncestor) throw new Error("candidate must be an ancestor of fixture")
+ if (fileMode !== "100644" && fileMode !== "100755") {
+ throw new Error("characterization fixture must modify a regular source file")
+ }
+ if (changes.length !== 1) throw new Error("characterization requires a single source change")
+ if (changes[0].status !== "M" || changes[0].old_path) {
+ throw new Error("characterization fixture must modify one existing source file")
+ }
+
+ const integrityKeys = Object.keys(integrityBlobs).sort()
+ const expectedIntegrity = [...manifest.integrity_paths].sort()
+ if (JSON.stringify(integrityKeys) !== JSON.stringify(expectedIntegrity)) {
+ throw new Error("characterization integrity blob set does not match the manifest")
+ }
+ for (const [path, blobs] of Object.entries(integrityBlobs)) {
+ if (blobs.candidate !== blobs.fixture) {
+ throw new Error(`characterization policy blob changed: ${path}`)
+ }
+ }
+
+ validateExecutionPlan(plan, { manifest })
+ if (!plan.diagnostic_only) throw new Error("characterization plan must be diagnostic-only")
+ if (plan.base_sha !== candidateSha || plan.head_sha !== fixtureSha) {
+ throw new Error("characterization plan SHA boundary mismatch")
+ }
+ if (plan.full || !CLASSES.has(plan.change_class) || plan.change_class !== expectedClass) {
+ throw new Error(`characterization classifier class mismatch: ${plan.change_class}`)
+ }
+ if (plan.coverage.required_changed_files.length !== 1
+ || plan.coverage.required_changed_files[0] !== changes[0].path) {
+ throw new Error("characterization fixture path must be coverable")
+ }
+
+ return {
+ schema_version: manifest.schema_version,
+ policy_version: manifest.policy_version,
+ base_sha: candidateSha,
+ head_sha: fixtureSha,
+ expected_class: expectedClass,
+ diagnostic_only: true,
+ source_path: changes[0].path,
+ plan_hash: plan.plan_hash,
+ integrity_paths: expectedIntegrity,
+ }
+}
+
+function git(args, options = {}, runtime = {}) {
+ const execute = runtime.execFileSync || execFileSync
+ return execute("git", args, {
+ cwd: runtime.root || ROOT,
+ ...(options.buffer ? {} : { encoding: "utf8" }),
+ stdio: ["ignore", "pipe", "pipe"],
+ })
+}
+
+export function blobAt(sha, path, runtime = {}) {
+ const spawn = runtime.spawnSync || spawnSync
+ const result = spawn("git", ["rev-parse", `${sha}:${path}`], {
+ cwd: runtime.root || ROOT,
+ encoding: "utf8",
+ stdio: ["ignore", "pipe", "pipe"],
+ })
+ if (result.status === 0) return result.stdout.trim()
+ if (result.stderr.includes("exists on disk, but not in")) return null
+ if (result.stderr.includes("does not exist in")) return null
+ throw new Error(`cannot resolve integrity path ${path}: ${result.stderr.trim()}`)
+}
+
+function parseArgs(argv) {
+ const args = {}
+ for (let index = 0; index < argv.length; index += 1) {
+ const key = argv[index].replace(/^--/, "").replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+ args[key] = argv[++index]
+ }
+ return args
+}
+
+export function runCli(argv, runtime = {}) {
+ const args = parseArgs(argv)
+ for (const required of [
+ "repository", "refName", "eventSha", "candidateSha", "fixtureSha",
+ "expectedClass", "candidatePrJson",
+ ]) {
+ if (!args[required]) throw new Error(`--${required} is required`)
+ }
+ const manifest = runtime.manifest || loadScopeManifest()
+ const candidatePr = JSON.parse(readFileSync(resolve(args.candidatePrJson), "utf8"))
+ const remoteRef = git(["ls-remote", "--heads", "origin", `refs/heads/${args.refName}`], {}, runtime)
+ .trim()
+ .split(/\s+/)[0]
+ const parentShas = git(["show", "-s", "--format=%P", args.fixtureSha], {}, runtime).trim().split(/\s+/).filter(Boolean)
+ const spawn = runtime.spawnSync || spawnSync
+ const ancestor = spawn("git", ["merge-base", "--is-ancestor", args.candidateSha, args.fixtureSha], {
+ cwd: runtime.root || ROOT,
+ stdio: "ignore",
+ }).status === 0
+ const changes = parseNameStatus(git(
+ ["diff", "--name-status", "-z", "--find-renames", "--find-copies", args.candidateSha, args.fixtureSha],
+ { buffer: true },
+ runtime,
+ ))
+ const fileMode = changes.length === 1
+ ? git(["ls-tree", args.fixtureSha, "--", changes[0].path], {}, runtime).trim().split(/\s+/)[0]
+ : ""
+ const plan = buildExecutionPlan(changes, {
+ baseSha: args.candidateSha,
+ headSha: args.fixtureSha,
+ diagnosticOnly: true,
+ })
+ const integrityBlobs = Object.fromEntries(manifest.integrity_paths.map((path) => [path, {
+ candidate: blobAt(args.candidateSha, path, runtime),
+ fixture: blobAt(args.fixtureSha, path, runtime),
+ }]))
+ const result = validateCharacterization({
+ repository: args.repository,
+ refName: args.refName,
+ eventSha: args.eventSha,
+ candidateSha: args.candidateSha,
+ fixtureSha: args.fixtureSha,
+ expectedClass: args.expectedClass,
+ candidatePr,
+ refSha: remoteRef,
+ parentShas,
+ isAncestor: ancestor,
+ fileMode,
+ changes,
+ plan,
+ integrityBlobs,
+ manifest,
+ })
+ if (args.output) {
+ appendFileSync(args.output, `base_sha=${result.base_sha}\nhead_sha=${result.head_sha}\ndiagnostic_only=true\n`)
+ }
+ if (args.summary) {
+ appendFileSync(
+ args.summary,
+ `## Characterization guard\n\n- Candidate: \`${result.base_sha}\`\n- Fixture: \`${result.head_sha}\`\n- Class: \`${result.expected_class}\`\n- Source: \`${result.source_path}\`\n- Policy blobs: \`${result.integrity_paths.length}\` identical\n`,
+ )
+ }
+ process.stdout.write(`${JSON.stringify(result, null, 2)}\n`)
+}
+
+export function runIfMain(
+ metaUrl,
+ argvPath = process.argv[1],
+ argv = process.argv.slice(2),
+ runtime = {},
+) {
+ if (argvPath === fileURLToPath(metaUrl)) {
+ runCli(argv, runtime)
+ }
+}
+
+runIfMain(import.meta.url)
diff --git a/scripts/ci/characterization-guard.test.ts b/scripts/ci/characterization-guard.test.ts
new file mode 100644
index 000000000..7a15991df
--- /dev/null
+++ b/scripts/ci/characterization-guard.test.ts
@@ -0,0 +1,250 @@
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { describe, expect, it, vi } from "vitest"
+import { buildExecutionPlan, loadScopeManifest } from "./changed-scopes.mjs"
+import {
+ blobAt,
+ runCli,
+ runIfMain,
+ validateCharacterization,
+} from "./characterization-guard.mjs"
+
+const candidateSha = "a".repeat(40)
+const fixtureSha = "b".repeat(40)
+const sourcePath = "src/cli/commands/update.ts"
+
+function validInput() {
+ const manifest = loadScopeManifest()
+ const changes = [{ status: "M", path: sourcePath }]
+ return {
+ repository: "alookai/alook",
+ refName: `ci-characterization/${candidateSha.slice(0, 12)}/package`,
+ eventSha: fixtureSha,
+ candidateSha,
+ fixtureSha,
+ expectedClass: "package",
+ candidatePr: {
+ state: "open",
+ head: { sha: candidateSha, repo: { full_name: "alookai/alook" } },
+ },
+ refSha: fixtureSha,
+ parentShas: [candidateSha],
+ isAncestor: true,
+ fileMode: "100644",
+ changes,
+ plan: buildExecutionPlan(changes, {
+ baseSha: candidateSha,
+ headSha: fixtureSha,
+ diagnosticOnly: true,
+ }),
+ integrityBlobs: Object.fromEntries(
+ manifest.integrity_paths.map((path: string) => [path, { candidate: "blob", fixture: "blob" }]),
+ ),
+ manifest,
+ }
+}
+
+function cliRuntime(sourcePath: string | null) {
+ const manifest = loadScopeManifest()
+ return {
+ manifest,
+ execFileSync: vi.fn((_file: string, args: string[], options: unknown) => {
+ if (args[0] === "ls-remote") return `${fixtureSha}\trefs/heads/test\n`
+ if (args[0] === "show") return `${candidateSha}\n`
+ if (args[0] === "diff") return sourcePath === null ? Buffer.alloc(0) : Buffer.from(`M\0${sourcePath}\0`)
+ if (args[0] === "ls-tree") return `100644 blob\t${sourcePath}\n`
+ throw new Error(`unexpected git call: ${args.join(" ")} / ${String(options)}`)
+ }),
+ spawnSync: vi.fn((_file: string, args: string[]) => {
+ if (args[0] === "merge-base") return { status: 0, stdout: "", stderr: "" }
+ if (args[0] === "rev-parse") return { status: 0, stdout: "policy-blob\n", stderr: "" }
+ throw new Error(`unexpected spawned git call: ${args.join(" ")}`)
+ }),
+ }
+}
+
+function cliArgs(candidatePrJson: string, expectedClass: string) {
+ return [
+ "--repository", "alookai/alook",
+ "--ref-name", `ci-characterization/${candidateSha.slice(0, 12)}/${expectedClass}`,
+ "--event-sha", fixtureSha,
+ "--candidate-sha", candidateSha,
+ "--fixture-sha", fixtureSha,
+ "--expected-class", expectedClass,
+ "--candidate-pr-json", candidatePrJson,
+ ]
+}
+
+describe("validateCharacterization", () => {
+ it("accepts only a locked same-repository direct-parent source fixture", () => {
+ const result = validateCharacterization(validInput())
+
+ expect(result).toMatchObject({
+ base_sha: candidateSha,
+ head_sha: fixtureSha,
+ expected_class: "package",
+ diagnostic_only: true,
+ source_path: sourcePath,
+ })
+ })
+
+ it.each([
+ ["SHA format", { eventSha: "A".repeat(40) }],
+ ["expected class", { expectedClass: "future" }],
+ ["fork", { candidatePr: { state: "open", head: { sha: candidateSha, repo: { full_name: "fork/alook" } } } }],
+ ["prefix", { refName: "feature/not-characterization" }],
+ ["event SHA", { eventSha: "c".repeat(40) }],
+ ["ref tip", { refSha: "c".repeat(40) }],
+ ["open", { candidatePr: { state: "closed", head: { sha: candidateSha, repo: { full_name: "alookai/alook" } } } }],
+ ["candidate head", { candidatePr: { state: "open", head: { sha: "c".repeat(40), repo: { full_name: "alookai/alook" } } } }],
+ ["direct parent", { parentShas: ["c".repeat(40)] }],
+ ["merge commit", { parentShas: [candidateSha, "c".repeat(40)] }],
+ ["ancestor", { isAncestor: false }],
+ ["regular source", { fileMode: "120000" }],
+ ["single source", { changes: [{ status: "M", path: sourcePath }, { status: "M", path: "src/cli/lib/config.ts" }] }],
+ ["deletion", { changes: [{ status: "D", path: sourcePath }] }],
+ ["class", { expectedClass: "shared" }],
+ ])("rejects invalid %s evidence", (_label, override) => {
+ expect(() => validateCharacterization({ ...validInput(), ...override })).toThrow()
+ })
+
+ it("rejects policy blob drift and non-coverable fixture paths", () => {
+ const missingIntegrity = validInput()
+ delete missingIntegrity.integrityBlobs[Object.keys(missingIntegrity.integrityBlobs)[0]]
+ expect(() => validateCharacterization(missingIntegrity)).toThrow("blob set")
+
+ const drift = validInput()
+ drift.integrityBlobs[".github/workflows/ci.yml"] = {
+ candidate: "one",
+ fixture: "two",
+ }
+ expect(() => validateCharacterization(drift)).toThrow("blob")
+
+ const testPath = "src/cli/commands/update.test.ts"
+ const nonCoverable = validInput()
+ nonCoverable.changes = [{ status: "M", path: testPath }]
+ nonCoverable.plan = buildExecutionPlan(nonCoverable.changes, {
+ baseSha: candidateSha,
+ headSha: fixtureSha,
+ diagnosticOnly: true,
+ })
+ expect(() => validateCharacterization(nonCoverable)).toThrow("coverable")
+
+ const blogContent = validInput()
+ const blogContentPath = "src/web/blog/src/content/example.mdx"
+ blogContent.expectedClass = "blog"
+ blogContent.refName = `ci-characterization/${candidateSha.slice(0, 12)}/blog`
+ blogContent.changes = [{ status: "M", path: blogContentPath }]
+ blogContent.plan = buildExecutionPlan(blogContent.changes, {
+ baseSha: candidateSha,
+ headSha: fixtureSha,
+ diagnosticOnly: true,
+ })
+ expect(() => validateCharacterization(blogContent)).toThrow("coverable")
+ })
+
+ it("rejects full, mixed, unknown, and non-diagnostic plans", () => {
+ for (const path of [
+ ".github/workflows/ci.yml",
+ "src/cli/commands/update.ts",
+ "unexpected/file.ts",
+ ]) {
+ const input = validInput()
+ input.changes = path === "src/cli/commands/update.ts"
+ ? [{ status: "M", path }, { status: "M", path: "src/shared/src/semver.ts" }]
+ : [{ status: "M", path }]
+ input.plan = buildExecutionPlan(input.changes, {
+ baseSha: candidateSha,
+ headSha: fixtureSha,
+ diagnosticOnly: path !== "unexpected/file.ts",
+ })
+ expect(() => validateCharacterization(input)).toThrow()
+ }
+
+ const boundary = validInput()
+ boundary.plan = buildExecutionPlan(boundary.changes, {
+ baseSha: "c".repeat(40),
+ headSha: fixtureSha,
+ diagnosticOnly: true,
+ })
+ expect(() => validateCharacterization(boundary)).toThrow("SHA boundary")
+ })
+})
+
+describe("characterization guard CLI", () => {
+ it.each([
+ ["package", "src/cli/src/commands/inbox.ts"],
+ ["blog", "src/web/blog/src/lib/posts.ts"],
+ ["shared", "src/shared/src/semver.ts"],
+ ])("emits locked %s evidence from the full command path", (expectedClass, sourcePath) => {
+ const directory = mkdtempSync(join(tmpdir(), "alook-characterization-"))
+ const candidatePrJson = join(directory, "candidate.json")
+ const output = join(directory, "output")
+ const summary = join(directory, "summary.md")
+ const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true)
+ try {
+ writeFileSync(candidatePrJson, JSON.stringify({
+ state: "open",
+ head: { sha: candidateSha, repo: { full_name: "alookai/alook" } },
+ }))
+ const args = [
+ ...cliArgs(candidatePrJson, expectedClass),
+ "--output", output,
+ "--summary", summary,
+ ]
+ const runtime = cliRuntime(sourcePath)
+ if (expectedClass === "package") {
+ runIfMain("file:///tmp/characterization-guard.mjs", "/tmp/characterization-guard.mjs", args, runtime)
+ } else {
+ runCli(args, runtime)
+ }
+
+ expect(readFileSync(output, "utf8")).toContain(`base_sha=${candidateSha}`)
+ expect(readFileSync(summary, "utf8")).toContain(`Class: \`${expectedClass}\``)
+ const result = JSON.parse(stdout.mock.calls.map(([value]) => String(value)).join(""))
+ expect(result).toMatchObject({ expected_class: expectedClass, source_path: sourcePath })
+ } finally {
+ stdout.mockRestore()
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+
+ it("requires the full locked argument set", () => {
+ expect(() => runCli([])).toThrow("--repository is required")
+ })
+
+ it("rejects an empty Git diff through its absent file mode", () => {
+ const directory = mkdtempSync(join(tmpdir(), "alook-characterization-empty-"))
+ const candidatePrJson = join(directory, "candidate.json")
+ try {
+ writeFileSync(candidatePrJson, JSON.stringify({
+ state: "open",
+ head: { sha: candidateSha, repo: { full_name: "alookai/alook" } },
+ }))
+ expect(() => runCli(
+ cliArgs(candidatePrJson, "package"),
+ cliRuntime(null),
+ )).toThrow("regular source file")
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+
+ it("maps present, absent, and invalid integrity blobs", () => {
+ expect(blobAt(candidateSha, "codecov.yml", {
+ spawnSync: () => ({ status: 0, stdout: "blob\n", stderr: "" }),
+ })).toBe("blob")
+ for (const stderr of [
+ "fatal: path exists on disk, but not in commit",
+ "fatal: path does not exist in commit",
+ ]) {
+ expect(blobAt(candidateSha, "codecov.yml", {
+ spawnSync: () => ({ status: 128, stdout: "", stderr }),
+ })).toBeNull()
+ }
+ expect(() => blobAt(candidateSha, "codecov.yml", {
+ spawnSync: () => ({ status: 128, stdout: "", stderr: "fatal: transport failed" }),
+ })).toThrow("cannot resolve integrity path")
+ })
+})
diff --git a/scripts/ci/e2e-shard-artifacts.mjs b/scripts/ci/e2e-shard-artifacts.mjs
index 2d6eced95..6fdfaa329 100644
--- a/scripts/ci/e2e-shard-artifacts.mjs
+++ b/scripts/ci/e2e-shard-artifacts.mjs
@@ -173,17 +173,22 @@ export function verifyArtifactClosure({
{ length: matrix.total },
(_, index) => `${SHARD_ARTIFACT_PREFIX}-${runId}-${index + 1}`,
).sort()
- const actualNames = readdirSync(root, { withFileTypes: true })
- .filter((entry) => entry.isDirectory())
- .map((entry) => entry.name)
- .sort()
- assertEqualArray(actualNames, expectedNames, "artifact directories must exactly match matrix shards")
+ const actualNames = readdirSync(root, { withFileTypes: true }).map((entry) => entry.name).sort()
+ const flatNames = ["blob-report", "e2e-shard-manifest"]
+ const flatSingleShard = matrix.total === 1
+ && actualNames.length === flatNames.length
+ && actualNames.every((name, index) => name === flatNames[index])
+ if (!flatSingleShard) {
+ assertEqualArray(actualNames, expectedNames, "artifact layout must exactly match matrix shards")
+ }
rmSync(output, { recursive: true, force: true })
mkdirSync(output, { recursive: true })
const manifests = []
for (let shard = 1; shard <= matrix.total; shard += 1) {
- const artifactDirectory = join(root, `${SHARD_ARTIFACT_PREFIX}-${runId}-${shard}`)
+ const artifactDirectory = flatSingleShard
+ ? root
+ : join(root, `${SHARD_ARTIFACT_PREFIX}-${runId}-${shard}`)
const files = walkFiles(artifactDirectory)
const manifestsFound = files.filter((path) => basename(path) === "shard-manifest.json")
const zips = files.filter((path) => path.endsWith(".zip"))
diff --git a/scripts/ci/e2e-shard-artifacts.test.ts b/scripts/ci/e2e-shard-artifacts.test.ts
index 0c5616ad5..fd245d3e3 100644
--- a/scripts/ci/e2e-shard-artifacts.test.ts
+++ b/scripts/ci/e2e-shard-artifacts.test.ts
@@ -20,6 +20,12 @@ const matrix = {
],
}
+const singleShardMatrix = {
+ include: [
+ { shard: 1, total: 1, specs: ["src/test/e2e-ui/a.spec.ts"] },
+ ],
+}
+
const roots: string[] = []
afterEach(() => {
vi.unstubAllGlobals()
@@ -48,6 +54,25 @@ function artifactRoot(attempts = [1, 2]) {
return root
}
+function flatArtifactRoot() {
+ const root = mkdtempSync(join(tmpdir(), "alook-flat-shard-artifact-"))
+ roots.push(root)
+ mkdirSync(join(root, "blob-report"), { recursive: true })
+ mkdirSync(join(root, "e2e-shard-manifest"), { recursive: true })
+ writeFileSync(join(root, "blob-report", "report-1.zip"), "zip-1")
+ writeFileSync(join(root, "e2e-shard-manifest", "shard-manifest.json"), JSON.stringify(
+ createShardManifest({
+ runId: "run-1",
+ attempt: 1,
+ sha: "abc123",
+ shard: 1,
+ total: 1,
+ specs: singleShardMatrix.include[0]!.specs,
+ }),
+ ))
+ return root
+}
+
describe("expectedMatrix", () => {
it.each([
["missing include", {}],
@@ -153,6 +178,62 @@ describe("resolveExecutedShards", () => {
})
describe("verifyArtifactClosure", () => {
+ it("accepts the flat download layout for exactly one expected shard", () => {
+ const root = flatArtifactRoot()
+ const output = join(root, "..", `${root.split("/").at(-1)}-merged`)
+ roots.push(output)
+
+ const result = verifyArtifactClosure({
+ root,
+ output,
+ matrix: singleShardMatrix,
+ runId: "run-1",
+ attempt: 1,
+ sha: "abc123",
+ executedShards: new Set([1]),
+ })
+
+ expect(result.expectedSpecs).toEqual(["a.spec.ts"])
+ expect(readFileSync(join(output, "shard-1-report-1.zip"), "utf8")).toBe("zip-1")
+ })
+
+ it("rejects flat multi-shard, mixed, and incomplete layouts", () => {
+ const multiShard = flatArtifactRoot()
+ expect(() => verifyArtifactClosure({
+ root: multiShard,
+ output: join(multiShard, "merged"),
+ matrix,
+ runId: "run-1",
+ attempt: 1,
+ sha: "abc123",
+ executedShards: new Set([1, 2]),
+ })).toThrow("exactly match matrix shards")
+
+ const mixed = flatArtifactRoot()
+ mkdirSync(join(mixed, "blob-report-run-1-1"))
+ expect(() => verifyArtifactClosure({
+ root: mixed,
+ output: join(mixed, "merged"),
+ matrix: singleShardMatrix,
+ runId: "run-1",
+ attempt: 1,
+ sha: "abc123",
+ executedShards: new Set([1]),
+ })).toThrow("exactly match matrix shards")
+
+ const incomplete = flatArtifactRoot()
+ rmSync(join(incomplete, "e2e-shard-manifest"), { recursive: true })
+ expect(() => verifyArtifactClosure({
+ root: incomplete,
+ output: join(incomplete, "merged"),
+ matrix: singleShardMatrix,
+ runId: "run-1",
+ attempt: 1,
+ sha: "abc123",
+ executedShards: new Set([1]),
+ })).toThrow("exactly match matrix shards")
+ })
+
it("accepts all previous-attempt artifacts for a merge-only rerun", () => {
const root = artifactRoot([1, 1])
const output = join(root, "..", `${root.split("/").at(-1)}-merged`)
diff --git a/scripts/ci/e2e-shards.mjs b/scripts/ci/e2e-shards.mjs
index b72acf645..223aaf0e9 100644
--- a/scripts/ci/e2e-shards.mjs
+++ b/scripts/ci/e2e-shards.mjs
@@ -151,7 +151,16 @@ export function planE2eShards(
}
export function createE2eMatrix(specs = discoverE2eSpecs(), image = resolvePlaywrightImage()) {
- const shards = planE2eShards(specs)
+ const inventory = discoverE2eSpecs()
+ if (specs.includes("all") && !(specs.length === 1 && specs[0] === "all")) {
+ throw new Error("the all sentinel cannot be combined with explicit specs")
+ }
+ const selected = specs.length === 1 && specs[0] === "all" ? inventory : specs
+ const unknown = selected.filter((spec) => !inventory.includes(spec))
+ if (unknown.length > 0) {
+ throw new Error(`specs are outside the UI inventory: ${unknown.join(", ")}`)
+ }
+ const shards = planE2eShards(selected)
return {
include: shards.map((shard) => ({
shard: shard.shard,
@@ -184,7 +193,11 @@ function writeSummary(path, matrix) {
export function runCli(argv) {
const args = parseArgs(argv)
- const matrix = createE2eMatrix()
+ const specs = args.specsJson ? JSON.parse(args.specsJson) : discoverE2eSpecs()
+ if (!Array.isArray(specs) || specs.some((spec) => typeof spec !== "string")) {
+ throw new Error("--specs-json must be a JSON array of spec paths")
+ }
+ const matrix = createE2eMatrix(specs)
const json = JSON.stringify(matrix)
if (args.output) appendFileSync(args.output, `e2e_matrix=${json}\n`)
if (args.summary) writeSummary(args.summary, matrix)
diff --git a/scripts/ci/e2e-shards.test.ts b/scripts/ci/e2e-shards.test.ts
index fc275ddf6..62fb13fdb 100644
--- a/scripts/ci/e2e-shards.test.ts
+++ b/scripts/ci/e2e-shards.test.ts
@@ -1,7 +1,7 @@
-import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
import { tmpdir } from "node:os"
import { join } from "node:path"
-import { describe, expect, it } from "vitest"
+import { describe, expect, it, vi } from "vitest"
import {
createE2eMatrix,
discoverE2eSpecs,
@@ -9,6 +9,7 @@ import {
planE2eShards,
resolvePlaywrightImage,
resolvePlaywrightVersion,
+ runCli,
} from "./e2e-shards.mjs"
describe("resolvePlaywrightVersion", () => {
@@ -96,24 +97,92 @@ describe("planE2eShards", () => {
describe("createE2eMatrix", () => {
it("emits shell-safe repo-local spec arguments", () => {
const matrix = createE2eMatrix([
- "a.spec.ts",
- "nested/b.spec.ts",
- "c.spec.ts",
- "d.spec.ts",
- "e.spec.ts",
+ "01-auth.spec.ts",
+ "02-server-channel-message.spec.ts",
+ "03-realtime-multiuser.spec.ts",
+ "04-dm.spec.ts",
+ "05-mention-bot.spec.ts",
])
const argumentsList = matrix.include.flatMap((entry) => entry.specs)
expect(argumentsList.sort()).toEqual([
- "src/test/e2e-ui/a.spec.ts",
- "src/test/e2e-ui/c.spec.ts",
- "src/test/e2e-ui/d.spec.ts",
- "src/test/e2e-ui/e.spec.ts",
- "src/test/e2e-ui/nested/b.spec.ts",
+ "src/test/e2e-ui/01-auth.spec.ts",
+ "src/test/e2e-ui/02-server-channel-message.spec.ts",
+ "src/test/e2e-ui/03-realtime-multiuser.spec.ts",
+ "src/test/e2e-ui/04-dm.spec.ts",
+ "src/test/e2e-ui/05-mention-bot.spec.ts",
])
expect(matrix.include.every((entry) => entry.total === E2E_SHARD_COUNT)).toBe(true)
expect(matrix.include.every(
(entry) => entry.image === "mcr.microsoft.com/playwright:v1.62.1-noble",
)).toBe(true)
})
+
+ it("creates one shard for the Blog contract and rejects specs outside inventory", () => {
+ const matrix = createE2eMatrix(["54-blog-multizone.spec.ts"])
+
+ expect(matrix.include).toHaveLength(1)
+ expect(matrix.include[0]).toMatchObject({
+ shard: 1,
+ total: 1,
+ predicted_seconds: 60,
+ specs: ["src/test/e2e-ui/54-blog-multizone.spec.ts"],
+ })
+ expect(() => createE2eMatrix(["future.spec.ts"])).toThrow("inventory")
+ })
+
+ it("expands the all sentinel to the exact live inventory", () => {
+ const matrix = createE2eMatrix(["all"])
+ const assigned = matrix.include.flatMap((entry) => entry.specs)
+
+ expect(assigned).toHaveLength(discoverE2eSpecs().length)
+ expect(new Set(assigned).size).toBe(assigned.length)
+ })
+
+ it("rejects the all sentinel when combined with explicit specs", () => {
+ expect(() => createE2eMatrix(["all", "54-blog-multizone.spec.ts"]))
+ .toThrow("all sentinel")
+ })
+})
+
+describe("E2E shard CLI", () => {
+ it("writes an explicit matrix and human-readable summary", () => {
+ const directory = mkdtempSync(join(tmpdir(), "alook-e2e-cli-"))
+ const output = join(directory, "output")
+ const summary = join(directory, "summary.md")
+ try {
+ runCli([
+ "--specs-json", JSON.stringify(["54-blog-multizone.spec.ts"]),
+ "--output", output,
+ "--summary", summary,
+ ])
+
+ expect(readFileSync(output, "utf8")).toContain(
+ "src/test/e2e-ui/54-blog-multizone.spec.ts",
+ )
+ expect(readFileSync(summary, "utf8")).toContain("| 1/1 | 60s |")
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+
+ it("prints the default live inventory without an output file", () => {
+ const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true)
+ try {
+ runCli([])
+
+ const matrix = JSON.parse(stdout.mock.calls.map(([value]) => String(value)).join(""))
+ expect(matrix.include.flatMap((entry: { specs: string[] }) => entry.specs))
+ .toHaveLength(discoverE2eSpecs().length)
+ } finally {
+ stdout.mockRestore()
+ }
+ })
+
+ it("rejects malformed specs JSON values", () => {
+ expect(() => runCli(["--specs-json", JSON.stringify({ spec: "54-blog-multizone.spec.ts" })]))
+ .toThrow("JSON array")
+ expect(() => runCli(["--specs-json", JSON.stringify([42])]))
+ .toThrow("JSON array")
+ })
})
diff --git a/scripts/ci/validate-execution-plan.mjs b/scripts/ci/validate-execution-plan.mjs
new file mode 100644
index 000000000..f34b6cffb
--- /dev/null
+++ b/scripts/ci/validate-execution-plan.mjs
@@ -0,0 +1,37 @@
+import { fileURLToPath } from "node:url"
+import { projectPlan, validateExecutionPlan } from "./changed-scopes.mjs"
+
+export function validatePlanEnvelope({ planJson, planHash, projectionName, projectionValue }) {
+ if (!planJson) throw new Error("CI execution plan is required")
+ const plan = validateExecutionPlan(JSON.parse(planJson))
+ if (plan.plan_hash !== planHash) throw new Error("CI execution plan envelope hash mismatch")
+
+ if (projectionName) {
+ const projections = projectPlan(plan)
+ if (!Object.hasOwn(projections, projectionName)) {
+ throw new Error(`unknown CI execution plan projection: ${projectionName}`)
+ }
+ if (projections[projectionName] !== projectionValue) {
+ throw new Error(`CI execution plan projection mismatch: ${projectionName}`)
+ }
+ }
+ return plan
+}
+
+export function runCli(env = process.env) {
+ const plan = validatePlanEnvelope({
+ planJson: env.CI_EXECUTION_PLAN,
+ planHash: env.CI_PLAN_HASH,
+ projectionName: env.CI_PROJECTION_NAME,
+ projectionValue: env.CI_PROJECTION_VALUE,
+ })
+ process.stdout.write(
+ `Validated execution plan ${plan.plan_hash} (schema ${plan.schema_version}, policy ${plan.policy_version}).\n`,
+ )
+}
+
+export function runIfMain(metaUrl, argvPath = process.argv[1], env = process.env) {
+ if (argvPath === fileURLToPath(metaUrl)) runCli(env)
+}
+
+runIfMain(import.meta.url)
diff --git a/scripts/ci/validate-execution-plan.test.ts b/scripts/ci/validate-execution-plan.test.ts
new file mode 100644
index 000000000..e241e5502
--- /dev/null
+++ b/scripts/ci/validate-execution-plan.test.ts
@@ -0,0 +1,62 @@
+import { describe, expect, it, vi } from "vitest"
+import { buildExecutionPlan, projectPlan, stablePlanJson } from "./changed-scopes.mjs"
+import { runCli, runIfMain, validatePlanEnvelope } from "./validate-execution-plan.mjs"
+
+const plan = buildExecutionPlan(
+ [{ status: "M", path: "src/cli/src/commands/inbox.ts" }],
+ { baseSha: "a".repeat(40), headSha: "b".repeat(40) },
+)
+
+describe("execution plan consumer envelope", () => {
+ it("validates the canonical hash and a mechanical projection", () => {
+ expect(validatePlanEnvelope({
+ planJson: stablePlanJson(plan),
+ planHash: plan.plan_hash,
+ projectionName: "integration_suites",
+ projectionValue: projectPlan(plan).integration_suites,
+ })).toEqual(plan)
+ })
+
+ it("rejects stale hashes, unknown projections, and altered selections", () => {
+ expect(() => validatePlanEnvelope({})).toThrow("required")
+ expect(() => validatePlanEnvelope({
+ planJson: stablePlanJson(plan),
+ planHash: "0".repeat(64),
+ })).toThrow("envelope hash")
+ expect(() => validatePlanEnvelope({
+ planJson: stablePlanJson(plan),
+ planHash: plan.plan_hash,
+ projectionName: "future_suite",
+ projectionValue: "[]",
+ })).toThrow("unknown")
+ expect(() => validatePlanEnvelope({
+ planJson: stablePlanJson(plan),
+ planHash: plan.plan_hash,
+ projectionName: "integration_suites",
+ projectionValue: "[]",
+ })).toThrow("projection mismatch")
+ })
+
+ it("runs the environment-backed consumer and reports the validated contract", () => {
+ const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true)
+ try {
+ const env = {
+ CI_EXECUTION_PLAN: stablePlanJson(plan),
+ CI_PLAN_HASH: plan.plan_hash,
+ CI_PROJECTION_NAME: "integration_suites",
+ CI_PROJECTION_VALUE: projectPlan(plan).integration_suites,
+ }
+ runCli(env)
+ runIfMain(
+ "file:///tmp/validate-execution-plan.mjs",
+ "/tmp/validate-execution-plan.mjs",
+ env,
+ )
+
+ expect(stdout).toHaveBeenCalledTimes(2)
+ expect(stdout).toHaveBeenCalledWith(expect.stringContaining(plan.plan_hash))
+ } finally {
+ stdout.mockRestore()
+ }
+ })
+})
diff --git a/scripts/ci/verify-coverage-plan.mjs b/scripts/ci/verify-coverage-plan.mjs
new file mode 100644
index 000000000..9a56db675
--- /dev/null
+++ b/scripts/ci/verify-coverage-plan.mjs
@@ -0,0 +1,118 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs"
+import { isAbsolute, relative, resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+import {
+ loadScopeManifest,
+ validateExecutionPlan,
+} from "./changed-scopes.mjs"
+
+const ROOT = resolve(fileURLToPath(new URL("../..", import.meta.url)))
+
+function normalizeReportPath(path, root) {
+ const normalized = path.replaceAll("\\", "/")
+ return (isAbsolute(normalized) ? relative(root, normalized) : normalized)
+ .replaceAll("\\", "/")
+ .replace(/^\.\//, "")
+}
+
+function statementCounts(file) {
+ const counts = Object.values(file.s || {})
+ return {
+ statements: counts.length,
+ covered: counts.filter((count) => Number(count) > 0).length,
+ }
+}
+
+export function verifyCoveragePlan(plan, report, options = {}) {
+ const root = options.root || ROOT
+ const manifest = options.manifest || loadScopeManifest()
+ validateExecutionPlan(plan, { manifest })
+
+ const files = Object.entries(report).map(([key, value]) => ({
+ path: normalizeReportPath(value.path || key, root),
+ coverage: value,
+ }))
+ const reportPaths = new Set(files.map((entry) => entry.path))
+
+ for (const path of plan.coverage.required_changed_files) {
+ if (!existsSync(resolve(root, path))) {
+ throw new Error(`required changed coverage file does not exist at head: ${path}`)
+ }
+ if (!reportPaths.has(path)) {
+ throw new Error(`required changed coverage file is missing from merged report: ${path}`)
+ }
+ }
+
+ const targets = {}
+ for (const targetName of plan.coverage.targets) {
+ const target = manifest.codecov_targets[targetName]
+ if (!target) throw new Error(`unknown Codecov project target: ${targetName}`)
+ const targetFiles = files.filter((entry) => (
+ entry.path === target.path || entry.path.startsWith(`${target.path}/`)
+ ))
+ const totals = targetFiles.reduce((sum, entry) => {
+ const counts = statementCounts(entry.coverage)
+ sum.statements += counts.statements
+ sum.covered += counts.covered
+ return sum
+ }, { statements: 0, covered: 0 })
+ if (targetFiles.length === 0 || totals.statements === 0) {
+ throw new Error(`Codecov project target ${targetName} requires a nonempty denominator`)
+ }
+ const percent = (totals.covered / totals.statements) * 100
+ if (percent < target.target) {
+ throw new Error(
+ `Codecov project target ${targetName} is ${percent.toFixed(2)}%, below ${target.target}%`,
+ )
+ }
+ targets[targetName] = {
+ path: target.path,
+ target: target.target,
+ files: targetFiles.length,
+ statements: totals.statements,
+ covered: totals.covered,
+ percent,
+ }
+ }
+
+ return {
+ schema_version: plan.schema_version,
+ policy_version: plan.policy_version,
+ plan_hash: plan.plan_hash,
+ include_roots: plan.coverage.include_roots,
+ required_changed_files: plan.coverage.required_changed_files,
+ report_files: [...reportPaths].sort(),
+ targets,
+ }
+}
+
+function parseArgs(argv) {
+ const args = {}
+ for (let index = 0; index < argv.length; index += 1) {
+ const key = argv[index].replace(/^--/, "").replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())
+ args[key] = argv[++index]
+ }
+ return args
+}
+
+export function runCli(argv) {
+ const args = parseArgs(argv)
+ if (!args.plan || !args.report || !args.output) {
+ throw new Error("--plan, --report, and --output are required")
+ }
+ const plan = JSON.parse(readFileSync(resolve(args.plan), "utf8"))
+ const report = JSON.parse(readFileSync(resolve(args.report), "utf8"))
+ const result = verifyCoveragePlan(plan, report)
+ writeFileSync(resolve(args.output), `${JSON.stringify(result, null, 2)}\n`)
+ process.stdout.write(
+ `Verified ${result.required_changed_files.length} changed coverage files across ${Object.keys(result.targets).length} Codecov project targets.\n`,
+ )
+}
+
+export function runIfMain(metaUrl, argvPath = process.argv[1], argv = process.argv.slice(2)) {
+ if (argvPath === fileURLToPath(metaUrl)) {
+ runCli(argv)
+ }
+}
+
+runIfMain(import.meta.url)
diff --git a/scripts/ci/verify-coverage-plan.test.ts b/scripts/ci/verify-coverage-plan.test.ts
new file mode 100644
index 000000000..0a9715c38
--- /dev/null
+++ b/scripts/ci/verify-coverage-plan.test.ts
@@ -0,0 +1,133 @@
+import { createHash } from "node:crypto"
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join, resolve } from "node:path"
+import { describe, expect, it } from "vitest"
+import { buildExecutionPlan, stablePlanJson } from "./changed-scopes.mjs"
+import { runCli, runIfMain, verifyCoveragePlan } from "./verify-coverage-plan.mjs"
+
+const baseSha = "a".repeat(40)
+const headSha = "b".repeat(40)
+
+function coveredFile(path: string, covered = true) {
+ return {
+ path,
+ statementMap: { "0": { start: { line: 1, column: 0 }, end: { line: 1, column: 1 } } },
+ s: { "0": covered ? 1 : 0 },
+ fnMap: {},
+ f: {},
+ branchMap: {},
+ b: {},
+ }
+}
+
+function resignPlan(value) {
+ const { plan_hash: _ignored, ...unsigned } = value
+ return {
+ ...value,
+ plan_hash: createHash("sha256").update(stablePlanJson(unsigned)).digest("hex"),
+ }
+}
+
+describe("verifyCoveragePlan", () => {
+ it("proves required changed files and a nonempty passing Codecov project denominator", () => {
+ const changed = "src/cli/commands/update.ts"
+ const plan = buildExecutionPlan([{ status: "M", path: changed }], { baseSha, headSha })
+ const report = {
+ [resolve(changed)]: coveredFile(resolve(changed)),
+ [resolve("src/cli/lib/config.ts")]: coveredFile(resolve("src/cli/lib/config.ts")),
+ }
+
+ const result = verifyCoveragePlan(plan, report)
+
+ expect(result.required_changed_files).toEqual([changed])
+ expect(result.targets.cli).toMatchObject({ files: 2, statements: 2, covered: 2, percent: 100 })
+ })
+
+ it("rejects a required surviving changed file missing from the merged report", () => {
+ const changed = "src/cli/commands/update.ts"
+ const plan = buildExecutionPlan([{ status: "M", path: changed }], { baseSha, headSha })
+
+ expect(() => verifyCoveragePlan(plan, {
+ [resolve("src/cli/lib/config.ts")]: coveredFile(resolve("src/cli/lib/config.ts")),
+ })).toThrow("required changed coverage file")
+ })
+
+ it("rejects a required changed file that is absent from the checked-out head", () => {
+ const changed = "src/cli/commands/update.ts"
+ const plan = buildExecutionPlan([{ status: "M", path: changed }], { baseSha, headSha })
+ const root = mkdtempSync(join(tmpdir(), "alook-coverage-head-"))
+ try {
+ expect(() => verifyCoveragePlan(plan, {
+ [changed]: coveredFile(changed),
+ }, { root })).toThrow("does not exist at head")
+ } finally {
+ rmSync(root, { recursive: true, force: true })
+ }
+ })
+
+ it("rejects empty and below-target Codecov project reports", () => {
+ const changed = "src/shared/src/semver.ts"
+ const plan = buildExecutionPlan([{ status: "M", path: changed }], { baseSha, headSha })
+ const testOnlyPlan = buildExecutionPlan(
+ [{ status: "M", path: "src/shared/src/logger.test.ts" }],
+ { baseSha, headSha },
+ )
+
+ expect(() => verifyCoveragePlan(testOnlyPlan, {})).toThrow("nonempty denominator")
+ expect(() => verifyCoveragePlan(plan, {
+ [resolve("src/cli/lib/config.ts")]: coveredFile(resolve("src/cli/lib/config.ts")),
+ [resolve("src/email-worker/src/index.ts")]: coveredFile(resolve("src/email-worker/src/index.ts")),
+ [resolve(changed)]: coveredFile(resolve(changed), false),
+ [resolve("src/web/src/lib/config.ts")]: coveredFile(resolve("src/web/src/lib/config.ts")),
+ [resolve("src/ws-do/src/index.ts")]: coveredFile(resolve("src/ws-do/src/index.ts")),
+ })).toThrow("project target shared")
+ })
+
+ it("rejects targets absent from the Codecov contract", () => {
+ const changed = "src/cli/commands/update.ts"
+ const plan = buildExecutionPlan([{ status: "M", path: changed }], { baseSha, headSha })
+ const invalid = structuredClone(plan)
+ invalid.coverage.targets = ["future"]
+
+ expect(() => verifyCoveragePlan(resignPlan(invalid), {
+ [resolve(changed)]: coveredFile(resolve(changed)),
+ })).toThrow("unknown Codecov project target")
+ })
+
+ it("writes an auditable target manifest from the CLI", () => {
+ const changed = "src/cli/commands/update.ts"
+ const plan = buildExecutionPlan([{ status: "M", path: changed }], { baseSha, headSha })
+ const directory = mkdtempSync(join(tmpdir(), "alook-coverage-plan-"))
+ const planPath = join(directory, "plan.json")
+ const reportPath = join(directory, "coverage.json")
+ const outputPath = join(directory, "manifest.json")
+ try {
+ writeFileSync(planPath, JSON.stringify(plan))
+ writeFileSync(reportPath, JSON.stringify({
+ [resolve(changed)]: coveredFile(resolve(changed)),
+ }))
+ const args = [
+ "--plan", planPath,
+ "--report", reportPath,
+ "--output", outputPath,
+ "--unused-value", "ignored",
+ ]
+ runIfMain(
+ "file:///tmp/verify-coverage-plan.mjs",
+ "/tmp/verify-coverage-plan.mjs",
+ args,
+ )
+ expect(JSON.parse(readFileSync(outputPath, "utf8"))).toMatchObject({
+ plan_hash: plan.plan_hash,
+ required_changed_files: [changed],
+ })
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+
+ it("requires every CLI input path", () => {
+ expect(() => runCli([])).toThrow("--plan, --report, and --output")
+ })
+})
diff --git a/scripts/ci/workflow-contract.test.ts b/scripts/ci/workflow-contract.test.ts
index 9594d733c..c1acaeec9 100644
--- a/scripts/ci/workflow-contract.test.ts
+++ b/scripts/ci/workflow-contract.test.ts
@@ -8,8 +8,9 @@ function normalizeWorkflow(text: string): string {
return text.replace(/\r\n/g, "\n")
}
-const workflow = normalizeWorkflow(readFileSync(resolve(workflowRoot, "e2e-ui.yml"), "utf8"))
const ciWorkflow = normalizeWorkflow(readFileSync(resolve(workflowRoot, "ci.yml"), "utf8"))
+const workflow = ciWorkflow
+const legacyUiWorkflowPath = resolve(workflowRoot, "e2e-ui.yml")
const playwrightConfig = readFileSync(
resolve(import.meta.dirname, "../../src/web/playwright.config.ts"),
"utf8",
@@ -189,12 +190,15 @@ function ciJob(name: string): string {
}
describe("E2E UI workflow", () => {
- it("runs before merge and manually without push or schedule triggers", () => {
+ it("uses the canonical CI scope before merge and removes the second classifier", () => {
expect(workflow).toMatch(/^ pull_request:/m)
expect(workflow).toMatch(/^ merge_group:/m)
expect(workflow).toMatch(/^ workflow_dispatch:/m)
- expect(workflow).not.toMatch(/^ push:/m)
+ expect(workflow).toMatch(/^ push:/m)
expect(workflow).not.toMatch(/^ schedule:/m)
+ expect(existsSync(legacyUiWorkflowPath)).toBe(false)
+ expect(workflow.match(/^ scope:$/gm)).toHaveLength(1)
+ expect(ciJob("e2e-ui")).toContain("needs.scope.outputs.e2e_matrix")
})
it("keeps failure diagnostics best-effort and merge inputs strict", () => {
@@ -215,9 +219,9 @@ describe("E2E UI workflow", () => {
expect(workflow).toContain("verify-merged")
expect(workflow).toContain("--reporter html,json")
expect(workflow).not.toContain("playwright-merge-runtime")
- expect(workflow.match(/pnpm install --frozen-lockfile/g)).toHaveLength(2)
- expect(workflow.match(/if-no-files-found: error/g)).toHaveLength(2)
- expect(workflow.match(/continue-on-error: true/g)).toHaveLength(1)
+ expect(ciJob("e2e-ui").match(/pnpm install --frozen-lockfile/g)).toHaveLength(1)
+ expect(ciJob("merge-reports").match(/pnpm install --frozen-lockfile/g)).toHaveLength(1)
+ expect(ciJob("e2e-ui").match(/continue-on-error: true/g)).toHaveLength(1)
})
it("reports shard failures directly to GitHub Checks", () => {
@@ -225,17 +229,18 @@ describe("E2E UI workflow", () => {
})
it("does not install Bun for Node-only browser tests", () => {
- expect(workflow).not.toContain("oven-sh/setup-bun")
+ expect(ciJob("e2e-ui")).not.toContain("oven-sh/setup-bun")
+ expect(ciJob("merge-reports")).not.toContain("oven-sh/setup-bun")
})
it("runs shards in the lockfile-selected Playwright image without host installs", () => {
expect(workflow).toContain("image: ${{ matrix.image }}")
expect(workflow).toContain("options: --init --ipc=host --user 1001")
expect(workflow).toMatch(/defaults:\n run:\n shell: bash/)
- expect(workflow).not.toContain("playwright-browser-cache")
- expect(workflow).not.toContain("~/.cache/ms-playwright")
- expect(workflow).not.toContain("playwright install-deps")
- expect(workflow).not.toContain("playwright install chromium")
+ expect(ciJob("e2e-ui")).not.toContain("playwright-browser-cache")
+ expect(ciJob("e2e-ui")).not.toContain("~/.cache/ms-playwright")
+ expect(ciJob("e2e-ui")).not.toContain("playwright install-deps")
+ expect(ciJob("e2e-ui")).not.toContain("playwright install chromium")
})
})
@@ -294,17 +299,79 @@ describe("CI workflow graph", () => {
expect(ciWorkflow).not.toMatch(/^ schedule:/m)
})
+ it("keeps manual dispatch full by default and guards diagnostic fixtures before scoping", () => {
+ const scope = ciJob("scope")
+ expect(ciWorkflow).toContain("options: [full, characterization]")
+ expect(ciWorkflow).toContain("default: full")
+ expect(ciWorkflow).toContain("pull-requests: read")
+ expect(scope).toContain('gh api "repos/$GITHUB_REPOSITORY/pulls/$CANDIDATE_PR"')
+ expect(scope).toContain("node scripts/ci/characterization-guard.mjs")
+ expect(scope).toContain('--ref-name "$GITHUB_REF_NAME"')
+ expect(scope).toContain('--candidate-sha "$CANDIDATE_SHA"')
+ expect(scope).toContain('--fixture-sha "$FIXTURE_SHA"')
+ expect(scope).toContain('--expected-class "$EXPECTED_CLASS"')
+ expect(scope).toContain('--diagnostic-only')
+ expect(scope).toContain('args+=(--base "$EVENT_SHA" --head "$EVENT_SHA" --force-full)')
+ expect(scope).not.toContain("ref: ${{ inputs.fixture_sha }}")
+ })
+
+ it("publishes one auditable plan and validates its envelope in every consumer", () => {
+ const scope = ciJob("scope")
+ expect(scope).toContain("--plan-file .ci/execution-plan.json")
+ expect(scope).toContain("name: execution-plan-${{ github.run_id }}-${{ github.run_attempt }}")
+ for (const job of [
+ "blog-build", "static-checks", "test-linux", "test-windows", "app-packed-artifact",
+ "e2e", "desktop-rust", "lighthouse", "e2e-ui", "merge-reports", "ci-gate", "ui-e2e-gate",
+ ]) {
+ const definition = ciJob(job)
+ expect(definition, job).toContain("CI_EXECUTION_PLAN: ${{ needs.scope.outputs.execution_plan }}")
+ expect(definition, job).toContain("CI_PLAN_HASH: ${{ needs.scope.outputs.plan_hash }}")
+ expect(definition, job).toContain("node scripts/ci/validate-execution-plan.mjs")
+ }
+ })
+
+ it("keeps named Codecov targets, patch enforcement, and changed-file proof auditable", () => {
+ const linux = ciJob("test-linux")
+ expect(linux).toContain("coverage_include_roots")
+ expect(linux).toContain("verify-coverage-plan.mjs")
+ expect(linux).toContain("coverage-target-manifest.json")
+ expect(linux).toContain("coverage/coverage-final.json")
+ expect(readFileSync(resolve(repositoryRoot, "codecov.yml"), "utf8")).toContain("target: 100%")
+ })
+
+ it("prepares shared services for every selected integration suite", () => {
+ const e2e = ciJob("e2e")
+ const webCondition = "if: contains(fromJSON(needs.scope.outputs.integration_suites), 'web')"
+ const cliCondition = "if: contains(fromJSON(needs.scope.outputs.integration_suites), 'cli')"
+ const daemonCondition = "if: contains(fromJSON(needs.scope.outputs.integration_suites), 'daemon')"
+
+ expect(e2e).toContain("if: needs.scope.outputs.run_e2e == 'true'")
+ expect(e2e).toContain("- name: Create local bindings\n run:")
+ expect(e2e).toContain("- run: pnpm run db:migrate")
+ expect(e2e).toContain("- name: Start dev servers\n run: |")
+ expect(e2e).toContain("- name: Wait for services\n run: |")
+ expect(e2e.match(new RegExp(webCondition.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g")))
+ .toHaveLength(1)
+ expect(e2e.match(new RegExp(cliCondition.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g")))
+ .toHaveLength(1)
+ expect(e2e.match(new RegExp(daemonCondition.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g")))
+ .toHaveLength(1)
+ expect(e2e).toContain(`${webCondition}\n run: pnpm test:e2e`)
+ expect(e2e).toContain(`${cliCondition}\n run: pnpm --filter @alook/cli run test:integration`)
+ expect(e2e).toContain(`${daemonCondition}\n run: pnpm --filter @alook/daemon run test:integration`)
+ })
+
it("consolidates static work and preserves non-blocking Knip steps", () => {
const staticChecks = ciJob("static-checks")
expect(ciWorkflow).not.toMatch(/^ quality:/m)
expect(ciWorkflow).not.toMatch(/^ knip:/m)
expect(ciWorkflow).not.toMatch(/^ build:/m)
expect(staticChecks.match(/pnpm install --frozen-lockfile/g)).toHaveLength(1)
- expect(staticChecks.match(/continue-on-error: true/g)).toHaveLength(2)
- expect(staticChecks).toContain("run: pnpm typecheck")
- expect(staticChecks).toContain("run: pnpm lint")
- expect(staticChecks).toContain("run: pnpm build")
- expect(staticChecks).toContain("run: pnpm knip")
+ expect(staticChecks.match(/continue-on-error: true/g)).toHaveLength(1)
+ expect(staticChecks).toContain("pnpm turbo run typecheck")
+ expect(staticChecks).toContain("pnpm turbo run lint")
+ expect(staticChecks).toContain("pnpm turbo run build")
+ expect(staticChecks).toContain("pnpm turbo run knip")
})
it("gates the consolidated jobs under the stable CI Gate", () => {
@@ -446,12 +513,10 @@ describe("CI test budgets", () => {
it("runs only Windows-relevant workspace packages after the process-bearing suites", () => {
const windows = ciJob("test-windows")
- expect(windows).toContain(
- "run: pnpm turbo run test --filter=@alook/cli --filter=@alook/app --filter=@alook/shared",
- )
- expect(windows).toContain(
- "run: pnpm turbo run test --affected --filter=@alook/cli --filter=@alook/app --filter=@alook/shared",
- )
+ expect(windows).toContain("for suite in app cli shared")
+ expect(windows).toContain('filters+=("--filter=@alook/$suite")')
+ expect(windows).toContain('pnpm turbo run test "${filters[@]}"')
+ expect(windows).not.toContain("--affected")
for (const packageName of ["web", "email-worker", "ws-do", "wake-worker"]) {
expect(windows).not.toContain(`--filter=@alook/${packageName}`)
}
@@ -568,7 +633,6 @@ describe("Native message external-link opener", () => {
describe("Turbo CI execution", () => {
const cachedJobs = ["blog-build", "static-checks", "test-linux", "test-windows"]
- const affectedJobs = ["static-checks", "test-windows"]
it("persists a task cache isolated by operating system, architecture, and job", () => {
expect(ciWorkflow.match(/actions\/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9/g))
@@ -585,41 +649,34 @@ describe("Turbo CI execution", () => {
}
})
- it("provides complete history and explicit comparison commits to affected jobs", () => {
+ it("passes explicit validated manifest selections instead of re-running an affected classifier", () => {
const scope = ciJob("scope")
expect(scope).toContain("full: ${{ steps.scope.outputs.full }}")
expect(scope).toContain("base_sha: ${{ steps.scope.outputs.base_sha }}")
expect(scope).toContain("head_sha: ${{ steps.scope.outputs.head_sha }}")
-
- for (const job of affectedJobs) {
- const definition = ciJob(job)
- expect(definition).toContain("fetch-depth: 0")
- expect(definition).toContain("needs.scope.outputs.full == 'true'")
- expect(definition).toContain("needs.scope.outputs.full != 'true'")
- expect(definition).toContain("TURBO_SCM_BASE: ${{ needs.scope.outputs.base_sha }}")
- expect(definition).toContain("TURBO_SCM_HEAD: ${{ needs.scope.outputs.head_sha }}")
- expect(definition).toContain("--affected")
- }
+ expect(scope).toContain("execution_plan: ${{ steps.scope.outputs.execution_plan }}")
+ expect(scope).toContain("plan_hash: ${{ steps.scope.outputs.plan_hash }}")
+ expect(ciWorkflow).not.toContain("--affected")
+ expect(ciJob("static-checks")).toContain("CI_PROJECTION_NAME: static_packages")
+ expect(ciJob("test-windows")).toContain("CI_PROJECTION_NAME: windows_suites")
})
it("retains full commands when scope classification fails closed", () => {
const staticChecks = ciJob("static-checks")
- expect(staticChecks).toContain("run: pnpm typecheck")
- expect(staticChecks).toContain("run: pnpm lint")
- expect(staticChecks).toContain("run: pnpm knip")
- expect(staticChecks).toContain(
- "run: pnpm build --filter=@alook/shared --filter=@alook/web --filter=@alook/cli --filter=@alook/email-worker --filter=@alook/ws-do --filter=@alook/wake-worker",
- )
+ expect(staticChecks).toContain("pnpm turbo run typecheck")
+ expect(staticChecks).toContain("pnpm turbo run lint")
+ expect(staticChecks).toContain("pnpm turbo run knip")
+ expect(staticChecks).toContain("SELECTED_PACKAGES: ${{ needs.scope.outputs.build_packages }}")
expect(staticChecks.match(/pnpm install --frozen-lockfile/g)).toHaveLength(1)
const linux = ciJob("test-linux")
- expect(linux).toContain("run: pnpm turbo run test --filter=@alook/daemon")
+ expect(linux).toContain("--project=daemon-node src/daemon")
const windows = ciJob("test-windows")
expect(windows).toContain("run: pnpm --filter @alook/daemon test")
- expect(linux).toContain("run: pnpm turbo run test --filter='!@alook/daemon'")
- expect(windows).toContain(
- "run: pnpm turbo run test --filter=@alook/cli --filter=@alook/app --filter=@alook/shared",
- )
- expect(linux.match(/VITEST_MAX_WORKERS: 1/g)).toHaveLength(1)
+ expect(linux).toContain('vitest_args=("${project_args[@]}")')
+ expect(linux).toContain('if (( ${#root_coverage[@]} )); then vitest_args+=("${root_coverage[@]}"); fi')
+ expect(linux).toContain('pnpm vitest run "${vitest_args[@]}"')
+ expect(windows).toContain("pnpm turbo run test \"${filters[@]}\"")
+ expect(linux.match(/VITEST_MAX_WORKERS=1/g)).toHaveLength(1)
expect(windows.match(/VITEST_MAX_WORKERS: 1/g)).toHaveLength(1)
})
@@ -629,6 +686,29 @@ describe("Turbo CI execution", () => {
expect(daemonVitestConfig).toContain("sequence: { groupOrder: 2 }")
})
+ it("builds agent-driver before every selected Linux consumer of its dist exports", () => {
+ const linux = ciJob("test-linux")
+ const buildStart = linux.indexOf("- name: Build selected agent-driver fixture")
+ const unitStart = linux.indexOf("- name: Run selected unit tests")
+ const realStart = linux.indexOf("- name: Run selected daemon real-process tests")
+ const packedStart = linux.indexOf("- name: Run selected package artifact tests")
+ const buildStep = linux.slice(buildStart, unitStart)
+
+ expect(buildStart).toBeGreaterThan(-1)
+ expect(buildStep).toContain("run: pnpm --filter @alook/agent-driver build")
+ for (const selector of [
+ "contains(fromJSON(needs.scope.outputs.unit_test_roots), 'src/daemon/agent-driver')",
+ "contains(fromJSON(needs.scope.outputs.unit_test_roots), 'src/daemon')",
+ "contains(fromJSON(needs.scope.outputs.linux_suites), 'agent-driver-packed')",
+ "contains(fromJSON(needs.scope.outputs.linux_suites), 'daemon-packed')",
+ "contains(fromJSON(needs.scope.outputs.linux_suites), 'daemon-real')",
+ ]) expect(buildStep).toContain(selector)
+ expect(buildStep).not.toContain("src/web")
+ expect(unitStart).toBeGreaterThan(buildStart)
+ expect(realStart).toBeGreaterThan(buildStart)
+ expect(packedStart).toBeGreaterThan(buildStart)
+ })
+
it("runs each direct Worker Node and runtime project once through its standard test task", () => {
const projectNames: string[] = []
for (const module of directWorkerModules) {
@@ -648,7 +728,7 @@ describe("Turbo CI execution", () => {
projectNames.push(nodeProject, runtimeProject)
}
expect(new Set(projectNames).size).toBe(projectNames.length)
- expect(ciJob("test-linux")).toContain("pnpm turbo run test --filter='!@alook/daemon'")
+ expect(ciJob("test-linux")).toContain("projects=(web-node web-runtime)")
})
it("collects Node and workerd projects in one Istanbul report", () => {
@@ -688,21 +768,52 @@ describe("Turbo CI execution", () => {
)
})
+ it("runs selected non-daemon roots in isolated reports with exact coverage includes", () => {
+ const linux = ciJob("test-linux")
+ expect(linux).toContain('for root in "${non_daemon[@]}"; do')
+ expect(linux).toContain('if [[ "$include" == "$root/"* ]]; then')
+ expect(linux).toContain('root_coverage+=("--coverage.include=$include")')
+ expect(linux).toContain("--coverage.reporter=json")
+ expect(linux).toContain(
+ '"--coverage.reportsDirectory=$coverage_dir/root-${root_index}-${report_stem}"',
+ )
+ for (const projects of [
+ "projects=(ci-scripts)",
+ "projects=(email-worker-node email-worker-runtime)",
+ "projects=(wake-worker-node wake-worker-runtime)",
+ "projects=(web-node web-runtime)",
+ "projects=(ws-do-node ws-do-runtime)",
+ ]) expect(linux).toContain(projects)
+ expect(linux).toContain('project_args+=("--project=$project")')
+ expect(linux).toContain('pnpm vitest run "${vitest_args[@]}"')
+ expect(linux).toContain(
+ '--outputFile.blob="$report_dir/root-${root_index}-${report_stem}.blob"',
+ )
+ expect(linux).not.toContain("--project='!daemon-node'")
+ expect(linux).not.toContain('"${non_daemon[@]}" "${coverage[@]}"')
+ expect(linux).not.toContain(".vitest-reports/non-daemon.blob")
+ })
+
it("keeps package builds away from dist consumers and uploads one merged Istanbul report", () => {
const linux = ciJob("test-linux")
expect(linux).toContain("timeout-minutes: 25")
expect(linux).toContain(
"RUN_COVERAGE: ${{ github.event_name != 'push' || startsWith(github.event.head_commit.message, 'release:') }}",
)
- expect(linux).toContain("pnpm vitest run --project='!daemon-node' --coverage")
- expect(linux).toContain("pnpm vitest run --project=daemon-node --coverage")
- expect(linux).toContain("--reporter=blob --outputFile=.vitest-reports/non-daemon.blob")
- expect(linux).toContain("--reporter=blob --outputFile=.vitest-reports/daemon.blob")
+ expect(linux).toContain('pnpm vitest run "${vitest_args[@]}"')
+ expect(linux).toContain("pnpm vitest run --project=daemon-node src/daemon")
+ expect(linux).toContain(
+ '"--coverage.reportsDirectory=$coverage_dir/root-${root_index}-${report_stem}"',
+ )
+ expect(linux).toContain('coverage=(--coverage)')
+ expect(linux).toContain('--reporter=default --reporter=blob')
+ expect(linux).toContain('--outputFile.blob="$report_dir/root-${root_index}-${report_stem}.blob"')
+ expect(linux).toContain('--outputFile.blob="$report_dir/daemon.blob"')
expect(linux).toContain("--merge-reports=.vitest-reports")
expect(linux.match(/codecov\/codecov-action/g)).toHaveLength(1)
expect(linux).toContain("if: env.RUN_COVERAGE == 'true'")
- expect(linux).toContain("if: env.RUN_COVERAGE != 'true'")
expect(linux).toContain("files: ./coverage/coverage-final.json")
+ expect(linux).toContain("verify-coverage-plan.mjs")
for (const testPath of [
"src/pack.test.ts",
"src/cli/daemonLifecycle.real.test.ts",
@@ -721,7 +832,7 @@ describe("Turbo CI execution", () => {
"src/cli/diagnosticsLifecycle.real.test.ts",
"src/cli/daemonStop.test.ts",
]) {
- expect(linux.slice(linux.indexOf("- name: Run daemon real-process tests after coverage")))
+ expect(linux.slice(linux.indexOf("- name: Run selected daemon real-process tests")))
.toContain(testPath)
}
expect(linux).toContain(
@@ -730,15 +841,13 @@ describe("Turbo CI execution", () => {
expect(linux).toContain(
"pnpm --filter @alook/daemon exec vitest run --no-file-parallelism src/version.packed.test.ts src/agent-driver-bundle.packed.test.ts src/cli/daemonSelfUpdate.real.test.ts",
)
- const nonDaemonCoverageRun = linux.indexOf("- name: Run non-daemon tests with coverage")
- const daemonCoverageRun = linux.indexOf("- name: Run daemon tests with coverage")
- const coverageMerge = linux.indexOf("- name: Merge coverage reports")
- const realProcessRuns = linux.indexOf("- name: Run daemon real-process tests after coverage")
- const packageRuns = linux.indexOf("- name: Run package artifact tests after coverage")
+ const selectedUnitRun = linux.indexOf("- name: Run selected unit tests")
+ const coverageMerge = linux.indexOf("- name: Merge and verify selected coverage")
+ const realProcessRuns = linux.indexOf("- name: Run selected daemon real-process tests")
+ const packageRuns = linux.indexOf("- name: Run selected package artifact tests")
const coverageUpload = linux.indexOf("- name: Upload coverage")
- expect(nonDaemonCoverageRun).toBeGreaterThan(-1)
- expect(daemonCoverageRun).toBeGreaterThan(nonDaemonCoverageRun)
- expect(coverageMerge).toBeGreaterThan(daemonCoverageRun)
+ expect(selectedUnitRun).toBeGreaterThan(-1)
+ expect(coverageMerge).toBeGreaterThan(selectedUnitRun)
expect(realProcessRuns).toBeGreaterThan(coverageMerge)
expect(packageRuns).toBeGreaterThan(realProcessRuns)
expect(coverageUpload).toBeGreaterThan(packageRuns)