Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): diff main's nightly verification against the cap-v0.6.0 baseline
Richie McIlroyed4d899deaa0
diff --git a/.github/workflows/baseline-diff.yml b/.github/workflows/baseline-diff.ymlnew file mode 100644index 000000000..141153a3d--- /dev/null+++ b/.github/workflows/baseline-diff.yml@@ -0,0 +1,115 @@+name: Baseline diff++# Compares main's nightly Engine verify and real-capture runs with the latest+# run of the same workflow on the production baseline branch+# (scratch/prod-baseline: cap-v0.6.0 plus the suite), case by case+# (scripts/verify/diff.py). Fails when main is worse than 0.6.0 anywhere. The+# baseline is re-run weekly so its report artifacts never expire.++on:+ workflow_run:+ workflows: ["Engine verify", "Verify recording (real capture)"]+ types: [completed]+ branches: [main]+ schedule:+ - cron: "17 2 * * 1"+ workflow_dispatch:+ inputs:+ main-run:+ description: "Run id on main to compare (Engine verify or real capture)"+ required: true+ baseline-run:+ description: "Baseline run id (default: the latest of the same workflow on scratch/prod-baseline)"+ default: ""++permissions:+ contents: read+ actions: write++concurrency:+ group: baseline-diff-${{ github.event.workflow_run.id || inputs.main-run || 'refresh' }}+ cancel-in-progress: false++env:+ BASELINE_BRANCH: scratch/prod-baseline++jobs:+ refresh:+ if: github.event_name == 'schedule'+ runs-on: blacksmith-2vcpu-ubuntu-2404+ steps:+ - name: Re-run the baseline+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ gh workflow run engine-verify.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" --raw-field tier=full+ gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH"++ diff:+ if: >-+ github.event_name == 'workflow_dispatch' ||+ (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule')+ runs-on: blacksmith-2vcpu-ubuntu-2404+ steps:+ - uses: actions/checkout@v4+ with:+ persist-credentials: false++ - name: Runs+ id: runs+ env:+ GH_TOKEN: ${{ github.token }}+ MAIN_RUN: ${{ github.event.workflow_run.id || inputs.main-run }}+ BASELINE_INPUT: ${{ inputs.baseline-run }}+ run: |+ set -euo pipefail+ workflow=$(gh run view "$MAIN_RUN" --repo "$GITHUB_REPOSITORY" --json workflowDatabaseId --jq .workflowDatabaseId)+ baseline="$BASELINE_INPUT"+ if [[ -z "$baseline" ]]; then+ baseline=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$workflow" --branch "$BASELINE_BRANCH" \+ --status completed --limit 1 --json databaseId --jq '.[0].databaseId // empty')+ fi+ if [[ -z "$baseline" ]]; then+ echo "::error::no completed baseline run of workflow $workflow on $BASELINE_BRANCH"+ exit 1+ fi+ echo "main=$MAIN_RUN" >> "$GITHUB_OUTPUT"+ echo "baseline=$baseline" >> "$GITHUB_OUTPUT"+ echo "Comparing main run $MAIN_RUN with baseline run $baseline" | tee -a "$GITHUB_STEP_SUMMARY"++ - name: Download reports+ env:+ GH_TOKEN: ${{ github.token }}+ MAIN_RUN: ${{ steps.runs.outputs.main }}+ BASELINE_RUN: ${{ steps.runs.outputs.baseline }}+ run: |+ set -euo pipefail+ for side in main baseline; do+ run=$([[ $side == main ]] && echo "$MAIN_RUN" || echo "$BASELINE_RUN")+ dest="$RUNNER_TEMP/reports/$side"+ mkdir -p "$dest"+ gh run download "$run" --repo "$GITHUB_REPOSITORY" --dir "$dest" \+ --pattern 'verify-report-*' --pattern 'verify-recording-*'+ # Recordings are large and not read by the diff.+ find "$dest" -type d -name verify-scratch -prune -exec rm -r {} ++ echo "$side: $(find "$dest" -name '*.jsonl' | wc -l) report files"+ done++ - name: Diff+ run: |+ python3 scripts/verify/diff.py \+ "$RUNNER_TEMP/reports/baseline" "$RUNNER_TEMP/reports/main" \+ --na scripts/verify/baseline-na.tsv \+ --title "cap-v0.6.0 baseline vs main" \+ --fail-on-worse > "$RUNNER_TEMP/baseline-diff.md"+ status=$?+ cat "$RUNNER_TEMP/baseline-diff.md" >> "$GITHUB_STEP_SUMMARY"+ exit "$status"+ shell: bash {0}++ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ if: always()+ with:+ name: baseline-diff+ path: ${{ runner.temp }}/baseline-diff.md+ if-no-files-found: ignorediff --git a/scripts/verify/baseline-na.tsv b/scripts/verify/baseline-na.tsvnew file mode 100644index 000000000..877252ee9--- /dev/null+++ b/scripts/verify/baseline-na.tsv@@ -0,0 +1,2 @@+# glob<TAB>reason (cases n/a on the cap-v0.6.0 production baseline)+captions/* caption projection model (synchronize_captions) and transcription audio assembly did not exist in cap-v0.6.0; captions suite not rundiff --git a/scripts/verify/diff.py b/scripts/verify/diff.pyindex d9332ad19..29ab169bf 100644--- a/scripts/verify/diff.py+++ b/scripts/verify/diff.py@@ -326,6 +326,7 @@ def main():ap.add_argument("--json")ap.add_argument("--title", default="baseline vs main")ap.add_argument("--na", help="file of 'case-glob<TAB>reason' lines: cases n/a on the baseline")+ ap.add_argument("--fail-on-worse", action="store_true", help="exit 1 when any lane has WORSE or junit WORSE rows")args = ap.parse_args()na = load_na(args.na)@@ -400,7 +401,18 @@ def main():if args.json:with open(args.json, "w") as fh:json.dump(out, fh, indent=1, default=list)+ if args.fail_on_worse:+ # A unit test that main no longer has was renamed or removed with the+ # code it tested; the case-level rows cover the product.+ def counted(items):+ return [item for item in items or [] if not any(r.startswith("missing on main (baseline") for r in item[1])]++ worse = sum(len(counted(lane.get("WORSE"))) + len(counted(lane.get("JUNIT-WORSE"))) for lane in out.values())+ if worse:+ print(f"\n{worse} row(s) worse than the baseline", file=sys.stderr)+ return 1+ return 0if __name__ == "__main__":- main()+ sys.exit(main())