Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): baseline refresh matches main's nightly shape, skip cancelled baselines, fail on missing lanes
Richie McIlroy6d549aef9afd
diff --git a/.github/workflows/baseline-diff.yml b/.github/workflows/baseline-diff.ymlindex 141153a3d..265573004 100644--- a/.github/workflows/baseline-diff.yml+++ b/.github/workflows/baseline-diff.yml@@ -1,10 +1,8 @@name: Baseline diff-# Compares main's nightly Engine verify and real-capture runs with the latest-# run of the same workflow on the production baseline branch-# (scratch/prod-baseline: cap-v0.6.0 plus the suite), case by case-# (scripts/verify/diff.py). Fails when main is worse than 0.6.0 anywhere. The-# baseline is re-run weekly so its report artifacts never expire.+# Report artifacts expire after 90 days and the baseline branch never+# changes, so a weekly cron re-runs the baseline in the same shape as main's+# nightly runs (FULL, three repeats of real capture on both runner pools).on:workflow_run:@@ -43,7 +41,8 @@ jobs:GH_TOKEN: ${{ github.token }}run: |gh workflow run engine-verify.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" --raw-field tier=full- gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH"+ gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" \+ --raw-field tier=full --raw-field repeat=3 --raw-field runners=bothdiff:if: >-@@ -66,8 +65,10 @@ jobs:workflow=$(gh run view "$MAIN_RUN" --repo "$GITHUB_REPOSITORY" --json workflowDatabaseId --jq .workflowDatabaseId)baseline="$BASELINE_INPUT"if [[ -z "$baseline" ]]; then+ # Baseline runs fail (0.6.0 has known bugs) but must not be cancelled.baseline=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$workflow" --branch "$BASELINE_BRANCH" \- --status completed --limit 1 --json databaseId --jq '.[0].databaseId // empty')+ --status completed --limit 20 --json databaseId,conclusion \+ --jq '[.[] | select(.conclusion != "cancelled")][0].databaseId // empty')fiif [[ -z "$baseline" ]]; thenecho "::error::no completed baseline run of workflow $workflow on $BASELINE_BRANCH"diff --git a/scripts/verify/diff.py b/scripts/verify/diff.pyindex 29ab169bf..bab9950ce 100644--- a/scripts/verify/diff.py+++ b/scripts/verify/diff.py@@ -408,6 +408,11 @@ def main():return [item for item in items or [] if not any(r.startswith("missing on main (baseline") for r in item[1])]worse = sum(len(counted(lane.get("WORSE"))) + len(counted(lane.get("JUNIT-WORSE"))) for lane in out.values())+ # A baseline run that stopped early leaves main's cases unmatched.+ unmatched = [lane for lane in lanes if mrec.get(lane) and not brec.get(lane)]+ for lane in unmatched:+ print(f"lane {lane}: no baseline records", file=sys.stderr)+ worse += len(unmatched)if worse:print(f"\n{worse} row(s) worse than the baseline", file=sys.stderr)return 1