Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): real capture on Blacksmith macOS behind an always-reported gate

Richie McIlroyfc9362633b9c
diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml
index e23087146..4ac253df0 100644
--- a/.github/workflows/verify-recording.yml
+++ b/.github/workflows/verify-recording.yml
@@ -1,14 +1,26 @@
name: Verify recording (real capture)
# Records a marker stimulus through Cap's real Studio and Instant pipelines
-# (ScreenCaptureKit + VideoToolbox) on hosted macOS runners and checks every
-# frame and audio marker against the host-clock truth. See
+# (ScreenCaptureKit + VideoToolbox) on macOS runners and checks every frame
+# and audio marker against the host-clock truth. See
# crates/verify/tests/recording.rs.
-
-# Pull requests (and pushes to the suite branches) that touch the capture,
-# encode, timestamp or verify code run the FAST tier once on macos-15; the
-# FULL tier runs on macos-15 and macos-26 nightly and on dispatch, and for a
-# pull request that touches a high-risk region of scripts/verify/guards.toml.
+#
+# Every pull request gets a "Verify recording" verdict (the job to require):
+# - Scope decides from the diff whether the change reaches the recorder's or
+# the stimulus's dependency closure or the harness (scripts/verify/plan.py
+# recording); a pull request that does not is green without running;
+# - one that does runs the FAST tier on Blacksmith macOS 15, or the FULL tier
+# on Blacksmith macOS 15 and 26 when it touches a high-risk region or an
+# engine crate; nightly and dispatch run FULL on both.
+#
+# Runners: Blacksmith macOS (M4 Pro VM, 6 vCPU, logged-in GUI session, no
+# concurrency cap). Hosted macOS is reserved for main and release CI and is
+# only used on dispatch (runners=hosted or both). Blacksmith VMs have no
+# VideoToolbox hardware H.264 encoder (probe run 38847791287), so recordings
+# there take the software (libx264) path; the hardware path is covered by
+# dispatches on hosted runners. Blacksmith needs Screen Recording and
+# Microphone granted in TCC, its display woken, and an audio device
+# (BlackHole); the setup step below does that.
on:
workflow_dispatch:
inputs:
@@ -25,47 +37,26 @@ on:
description: "Preserve recordings and raw crash snapshots for passing cases"
type: boolean
default: false
+ system_audio:
+ description: "Blacksmith only: play and require system audio through BlackHole (not yet characterised)"
+ type: boolean
+ default: false
+ runners:
+ description: "blacksmith (default), hosted (GitHub macOS, reserved for main and release) or both"
+ type: choice
+ options: [blacksmith, hosted, both]
+ default: blacksmith
schedule:
- cron: "41 3 * * *"
pull_request:
- paths:
- - "crates/recording/**"
- - "crates/scap-*/**"
- - "crates/enc-avfoundation/**"
- - "crates/timestamp/**"
- - "crates/camera*/**"
- - "crates/media-info/**"
- - "crates/verify/**"
- - "crates/project/**"
- - "Cargo.lock"
- - ".github/workflows/verify-recording.yml"
- - "crates/enc-ffmpeg/**"
- - "crates/audio/**"
- - "crates/editor/src/audio*.rs"
- - "crates/rendering/src/decoder/**"
- - "crates/video-decode/**"
- - "apps/desktop-gpui/src/upload*"
- - "apps/desktop-gpui/src/upload/**"
- - "scripts/verify/guards.toml"
-
+ types: [opened, synchronize, reopened, ready_for_review]
push:
branches:
- ci/verify-ci
- ci/verify-recording
- paths:
- - "crates/recording/**"
- - "crates/scap-*/**"
- - "crates/enc-avfoundation/**"
- - "crates/timestamp/**"
- - "crates/camera*/**"
- - "crates/media-info/**"
- - "crates/verify/**"
- - "crates/project/**"
- - "Cargo.lock"
- - ".github/workflows/verify-recording.yml"
concurrency:
- group: verify-recording-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name }}
+ group: verify-recording-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && format('-{0}', github.run_id) || '' }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
permissions:
@@ -78,42 +69,71 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
+ run: ${{ steps.scope.outputs.run }}
runners: ${{ steps.scope.outputs.runners }}
tier: ${{ steps.scope.outputs.tier }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- sparse-checkout: scripts/verify
+
- name: Scope
id: scope
env:
GH_TOKEN: ${{ github.token }}
EVENT: ${{ github.event_name }}
PR: ${{ github.event.pull_request.number }}
+ BEFORE: ${{ github.event.before }}
INPUT_TIER: ${{ inputs.tier }}
+ INPUT_RUNNERS: ${{ inputs.runners }}
+ RUSTUP_TOOLCHAIN: stable
run: |
set -euo pipefail
- both='["macos-15","macos-26"]'
+ files="$RUNNER_TEMP/changed.txt"
+ blacksmith='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]'
case "$EVENT" in
- schedule) runners="$both"; tier=full ;;
- workflow_dispatch) runners="$both"; tier="${INPUT_TIER:-full}" ;;
- pull_request)
- gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
- python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt"
- if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt"; then
- runners="$both"; tier=full
+ schedule) run=true; runners="$blacksmith"; tier=full ;;
+ workflow_dispatch)
+ run=true
+ tier="${INPUT_TIER:-full}"
+ case "${INPUT_RUNNERS:-blacksmith}" in
+ hosted) runners='["macos-15","macos-26"]' ;;
+ both) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' ;;
+ *) runners="$blacksmith" ;;
+ esac ;;
+ *)
+ if [[ "$EVENT" == pull_request ]]; then
+ gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$files"
+ max=3000
+ elif [[ "$BEFORE" =~ ^0+$ ]]; then
+ : > "$files"; echo Cargo.toml >> "$files"
+ max=300
else
- runners='["macos-15"]'; tier=fast
+ gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$GITHUB_SHA" --jq '.files[].filename' > "$files"
+ max=300
+ fi
+ python3 scripts/verify/plan.py recording --files "$files" --max-files "$max" > "$RUNNER_TEMP/scope.txt"
+ cat "$RUNNER_TEMP/scope.txt"
+ run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")"
+ python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt"
+ if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; then
+ runners="$blacksmith"; tier=full
+ else
+ runners='["blacksmith-6vcpu-macos-15"]'; tier=fast
fi ;;
- *) runners='["macos-15"]'; tier=fast ;;
esac
+ echo "run=$run" >> "$GITHUB_OUTPUT"
echo "runners=$runners" >> "$GITHUB_OUTPUT"
echo "tier=$tier" >> "$GITHUB_OUTPUT"
- echo "real capture: $tier tier on $runners" | tee -a "$GITHUB_STEP_SUMMARY"
+ if [[ "$run" == true ]]; then
+ echo "real capture: $tier tier on $runners" | tee -a "$GITHUB_STEP_SUMMARY"
+ else
+ echo "real capture: no recording code changed, nothing to run" | tee -a "$GITHUB_STEP_SUMMARY"
+ fi
real-capture:
needs: scope
+ if: needs.scope.outputs.run == 'true'
strategy:
fail-fast: false
matrix:
@@ -130,7 +150,21 @@ jobs:
CAP_VERIFY_SCRATCH: ${{ github.workspace }}/verify-scratch
CAP_VERIFY_REPORT_DIR: ${{ github.workspace }}/verify-report
CAP_VERIFY_DUMP: ${{ github.workspace }}/verify-dump
- CAP_VERIFY_AUDIO_OUTPUT: ${{ matrix.runner == 'macos-26' && 'Null Audio Device' || '' }}
+ CAP_VERIFY_HOST: ${{ startsWith(matrix.runner, 'blacksmith-') && 'blacksmith' || 'github-hosted' }}
+ # A case this runner cannot run fails, except system audio as the only
+ # audio while the stimulus stays silent (see below).
+ CAP_VERIFY_REQUIRE_SUPPORTED: "1"
+ CAP_VERIFY_UNSUPPORTED_ALLOW: ${{ !(startsWith(matrix.runner, 'blacksmith-') && inputs.system_audio) && 'recording/instant/system-only' || '' }}
+ # Hosted runners' output devices give ScreenCaptureKit only silence, so
+ # system audio is not checked there. Blacksmith gets BlackHole (set up
+ # below), which does carry the stimulus's audio on macOS 15 (not on
+ # 26), but its system-audio timing is not characterised yet (run
+ # 38843005408: system audio +31 to +44 ms against the mic, BlackHole
+ # latency not in the truth), so by default the stimulus stays silent
+ # there too; dispatch system_audio=true plays and requires it.
+ CAP_VERIFY_AUDIO_OUTPUT: ${{ startsWith(matrix.runner, 'blacksmith-') && 'BlackHole 2ch' || matrix.runner == 'macos-26' && 'Null Audio Device' || '' }}
+ CAP_VERIFY_STIMULUS_AUDIO: ${{ startsWith(matrix.runner, 'blacksmith-') && !inputs.system_audio && '0' || '' }}
+ CAP_VERIFY_REQUIRE_SYSTEM_AUDIO: ${{ startsWith(matrix.runner, 'blacksmith-') && inputs.system_audio && '1' || '' }}
steps:
- uses: actions/checkout@v4
@@ -174,6 +208,78 @@ jobs:
cargo test --locked -p cap-verify --test recording --no-run
echo "build seconds: $(( $(date +%s) - start ))"
+ # Blacksmith runs a logged-in GUI session with SIP disabled, but:
+ # - ScreenCaptureKit is denied until TCC grants the responsible process
+ # (Runner.Worker), so Screen Recording and Microphone are granted to
+ # every binary that captures plus the whole runner ancestry, in the
+ # system and the user TCC database (Microphone is a per-user
+ # service: without it CoreAudio blocks the stimulus forever in
+ # AudioDeviceCreateIOProcID waiting for coreaudiod), then tccd
+ # restarts;
+ # - the display starts asleep (CGDisplayIsAsleep, no active display),
+ # so ScreenCaptureKit lists windows but no display: wake it and keep
+ # it awake for the job;
+ # - there is no audio device: BlackHole provides input and output.
+ - name: Blacksmith capture permissions and audio device
+ if: startsWith(matrix.runner, 'blacksmith-')
+ run: |
+ set -euo pipefail
+ sudo pmset -a displaysleep 0 sleep 0 disksleep 0
+ caffeinate -u -t 2
+ nohup caffeinate -dimsu > /dev/null 2>&1 &
+ brew install --cask blackhole-2ch
+ brew install switchaudio-osx
+ sudo killall coreaudiod || true
+ for i in $(seq 1 30); do
+ SwitchAudioSource -a -t output | grep -qx "BlackHole 2ch" && break
+ sleep 1
+ done
+ SwitchAudioSource -s "BlackHole 2ch" -t output
+ SwitchAudioSource -s "BlackHole 2ch" -t input
+ echo "outputs:"; SwitchAudioSource -a -t output
+ echo "default output: $(SwitchAudioSource -c -t output)"
+
+ { cargo build --locked -p cap-verify-stimulus --message-format=json 2>/dev/null; \
+ cargo test --locked -p cap-verify --test recording --no-run --message-format=json 2>/dev/null; } \
+ | python3 -c 'import json, sys
+ for line in sys.stdin:
+ m = json.loads(line)
+ if m.get("reason") == "compiler-artifact" and m.get("executable") and m["target"]["name"] in ("recording", "cap-verify-stimulus"):
+ print(m["executable"])' > "$RUNNER_TEMP/capture-binaries"
+ test "$(wc -l < "$RUNNER_TEMP/capture-binaries")" -ge 2
+ p=$$
+ while [ "$p" -gt 1 ]; do
+ ps -o comm= -p "$p"
+ p=$(ps -o ppid= -p "$p" | tr -d ' ')
+ done >> "$RUNNER_TEMP/capture-binaries"
+ for c in "$(rustup which cargo)" "$HOME/.cargo/bin/cargo"; do echo "$c"; done >> "$RUNNER_TEMP/capture-binaries"
+
+ db="/Library/Application Support/com.apple.TCC/TCC.db"
+ user_db="$HOME/Library/Application Support/com.apple.TCC/TCC.db"
+ approvals="$HOME/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist"
+ mkdir -p "$(dirname "$approvals")"
+ now=$(date +%s)
+ until=$(date -u -v+1y '+%a %b %d %H:%M:%S GMT %Y')
+ sort -u "$RUNNER_TEMP/capture-binaries" | grep '^/' | while read -r client; do
+ for service in kTCCServiceScreenCapture kTCCServiceMicrophone; do
+ row="INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);"
+ sudo sqlite3 "$db" "$row"
+ sqlite3 "$user_db" "$row"
+ done
+ /usr/libexec/PlistBuddy -c "Add :$client date $until" "$approvals" >/dev/null 2>&1 || true
+ echo "granted $client"
+ done
+ killall tccd || true
+ sudo killall tccd || true
+ sleep 2
+ cat > "$RUNNER_TEMP/display.swift" <<'SWIFT'
+ import CoreGraphics
+ let main = CGMainDisplayID()
+ print("display \(main): asleep \(CGDisplayIsAsleep(main)), active \(CGDisplayIsActive(main)), \(CGDisplayBounds(main)), screen capture \(CGPreflightScreenCaptureAccess())")
+ SWIFT
+ swiftc "$RUNNER_TEMP/display.swift" -o "$RUNNER_TEMP/display"
+ "$RUNNER_TEMP/display"
+
- name: Real-capture recording oracle
run: |
mkdir -p "$CAP_VERIFY_DUMP"
@@ -196,6 +302,11 @@ jobs:
echo "test seconds: $(( $(date +%s) - start ))"
exit "$status"
+ - name: Summary
+ if: always()
+ run: |
+ python3 scripts/verify/summary.py --title "real capture ${{ matrix.runner }} ${{ needs.scope.outputs.tier }}" "$CAP_VERIFY_REPORT_DIR" >> "$GITHUB_STEP_SUMMARY" || true
+
- name: Upload artifacts
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -206,3 +317,56 @@ jobs:
${{ github.workspace }}/verify-dump
${{ github.workspace }}/verify-scratch
if-no-files-found: ignore
+
+ # The check to require: green when the change is out of scope or every
+ # planned runner passed with at least the tier's case count.
+ gate:
+ name: Verify recording
+ needs: [scope, real-capture]
+ if: always()
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ sparse-checkout: scripts/verify
+
+ - uses: actions/download-artifact@v4
+ if: needs.scope.outputs.run == 'true'
+ with:
+ pattern: verify-recording-*
+ path: ${{ runner.temp }}/reports
+
+ - name: Verdict
+ env:
+ SCOPE: ${{ needs.scope.result }}
+ RUN: ${{ needs.scope.outputs.run }}
+ CAPTURE: ${{ needs.real-capture.result }}
+ TIER: ${{ needs.scope.outputs.tier }}
+ RUNNERS: ${{ needs.scope.outputs.runners }}
+ CASE_FILTER: ${{ inputs.case_filter }}
+ run: |
+ if [[ "$SCOPE" != success ]]; then
+ echo "scoping failed" >&2
+ exit 1
+ fi
+ if [[ "$RUN" != true ]]; then
+ echo "No recording code changed: nothing to verify." | tee -a "$GITHUB_STEP_SUMMARY"
+ exit 0
+ fi
+ if [[ "$CAPTURE" != success ]]; then
+ echo "real capture $CAPTURE ($TIER tier): see the runner jobs" >&2
+ exit 1
+ fi
+ if [[ -n "$CASE_FILTER" ]]; then
+ echo "case filter set: executed-case floor not applied"
+ exit 0
+ fi
+ floor=0
+ python3 scripts/verify/floor.py --recording --tier "$TIER" --runners "$RUNNERS" "$RUNNER_TEMP/reports" > "$RUNNER_TEMP/floor.txt" || floor=$?
+ cat "$RUNNER_TEMP/floor.txt" | tee -a "$GITHUB_STEP_SUMMARY"
+ if [[ $floor != 0 ]]; then
+ echo "::error::fewer real-capture cases ran than planned"
+ exit 1
+ fi