Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): diff main's nightly verification against the cap-v0.6.0 baseline

Richie McIlroyed4d899deaa0
diff --git a/.github/workflows/baseline-diff.yml b/.github/workflows/baseline-diff.yml
new file mode 100644
index 000000000..141153a3d
--- /dev/null
+++ b/.github/workflows/baseline-diff.yml
@@ -0,0 +1,115 @@
+name: Baseline diff
+
+# Compares main's nightly Engine verify and real-capture runs with the latest
+# run of the same workflow on the production baseline branch
+# (scratch/prod-baseline: cap-v0.6.0 plus the suite), case by case
+# (scripts/verify/diff.py). Fails when main is worse than 0.6.0 anywhere. The
+# baseline is re-run weekly so its report artifacts never expire.
+
+on:
+ workflow_run:
+ workflows: ["Engine verify", "Verify recording (real capture)"]
+ types: [completed]
+ branches: [main]
+ schedule:
+ - cron: "17 2 * * 1"
+ workflow_dispatch:
+ inputs:
+ main-run:
+ description: "Run id on main to compare (Engine verify or real capture)"
+ required: true
+ baseline-run:
+ description: "Baseline run id (default: the latest of the same workflow on scratch/prod-baseline)"
+ default: ""
+
+permissions:
+ contents: read
+ actions: write
+
+concurrency:
+ group: baseline-diff-${{ github.event.workflow_run.id || inputs.main-run || 'refresh' }}
+ cancel-in-progress: false
+
+env:
+ BASELINE_BRANCH: scratch/prod-baseline
+
+jobs:
+ refresh:
+ if: github.event_name == 'schedule'
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ steps:
+ - name: Re-run the baseline
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ gh workflow run engine-verify.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" --raw-field tier=full
+ gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH"
+
+ diff:
+ if: >-
+ github.event_name == 'workflow_dispatch' ||
+ (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule')
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+
+ - name: Runs
+ id: runs
+ env:
+ GH_TOKEN: ${{ github.token }}
+ MAIN_RUN: ${{ github.event.workflow_run.id || inputs.main-run }}
+ BASELINE_INPUT: ${{ inputs.baseline-run }}
+ run: |
+ set -euo pipefail
+ workflow=$(gh run view "$MAIN_RUN" --repo "$GITHUB_REPOSITORY" --json workflowDatabaseId --jq .workflowDatabaseId)
+ baseline="$BASELINE_INPUT"
+ if [[ -z "$baseline" ]]; then
+ baseline=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$workflow" --branch "$BASELINE_BRANCH" \
+ --status completed --limit 1 --json databaseId --jq '.[0].databaseId // empty')
+ fi
+ if [[ -z "$baseline" ]]; then
+ echo "::error::no completed baseline run of workflow $workflow on $BASELINE_BRANCH"
+ exit 1
+ fi
+ echo "main=$MAIN_RUN" >> "$GITHUB_OUTPUT"
+ echo "baseline=$baseline" >> "$GITHUB_OUTPUT"
+ echo "Comparing main run $MAIN_RUN with baseline run $baseline" | tee -a "$GITHUB_STEP_SUMMARY"
+
+ - name: Download reports
+ env:
+ GH_TOKEN: ${{ github.token }}
+ MAIN_RUN: ${{ steps.runs.outputs.main }}
+ BASELINE_RUN: ${{ steps.runs.outputs.baseline }}
+ run: |
+ set -euo pipefail
+ for side in main baseline; do
+ run=$([[ $side == main ]] && echo "$MAIN_RUN" || echo "$BASELINE_RUN")
+ dest="$RUNNER_TEMP/reports/$side"
+ mkdir -p "$dest"
+ gh run download "$run" --repo "$GITHUB_REPOSITORY" --dir "$dest" \
+ --pattern 'verify-report-*' --pattern 'verify-recording-*'
+ # Recordings are large and not read by the diff.
+ find "$dest" -type d -name verify-scratch -prune -exec rm -r {} +
+ echo "$side: $(find "$dest" -name '*.jsonl' | wc -l) report files"
+ done
+
+ - name: Diff
+ run: |
+ python3 scripts/verify/diff.py \
+ "$RUNNER_TEMP/reports/baseline" "$RUNNER_TEMP/reports/main" \
+ --na scripts/verify/baseline-na.tsv \
+ --title "cap-v0.6.0 baseline vs main" \
+ --fail-on-worse > "$RUNNER_TEMP/baseline-diff.md"
+ status=$?
+ cat "$RUNNER_TEMP/baseline-diff.md" >> "$GITHUB_STEP_SUMMARY"
+ exit "$status"
+ shell: bash {0}
+
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ if: always()
+ with:
+ name: baseline-diff
+ path: ${{ runner.temp }}/baseline-diff.md
+ if-no-files-found: ignore
diff --git a/scripts/verify/baseline-na.tsv b/scripts/verify/baseline-na.tsv
new file mode 100644
index 000000000..877252ee9
--- /dev/null
+++ b/scripts/verify/baseline-na.tsv
@@ -0,0 +1,2 @@
+# glob<TAB>reason (cases n/a on the cap-v0.6.0 production baseline)
+captions/* caption projection model (synchronize_captions) and transcription audio assembly did not exist in cap-v0.6.0; captions suite not run
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.py
index d9332ad19..29ab169bf 100644
--- a/scripts/verify/diff.py
+++ b/scripts/verify/diff.py
@@ -326,6 +326,7 @@ def main():
ap.add_argument("--json")
ap.add_argument("--title", default="baseline vs main")
ap.add_argument("--na", help="file of 'case-glob<TAB>reason' lines: cases n/a on the baseline")
+ ap.add_argument("--fail-on-worse", action="store_true", help="exit 1 when any lane has WORSE or junit WORSE rows")
args = ap.parse_args()
na = load_na(args.na)
@@ -400,7 +401,18 @@ def main():
if args.json:
with open(args.json, "w") as fh:
json.dump(out, fh, indent=1, default=list)
+ if args.fail_on_worse:
+ # A unit test that main no longer has was renamed or removed with the
+ # code it tested; the case-level rows cover the product.
+ def counted(items):
+ return [item for item in items or [] if not any(r.startswith("missing on main (baseline") for r in item[1])]
+
+ worse = sum(len(counted(lane.get("WORSE"))) + len(counted(lane.get("JUNIT-WORSE"))) for lane in out.values())
+ if worse:
+ print(f"\n{worse} row(s) worse than the baseline", file=sys.stderr)
+ return 1
+ return 0
if __name__ == "__main__":
- main()
+ sys.exit(main())