Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): real capture on Blacksmith macOS behind an always-reported gate
Richie McIlroyfc9362633b9c
diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.ymlindex e23087146..4ac253df0 100644--- a/.github/workflows/verify-recording.yml+++ b/.github/workflows/verify-recording.yml@@ -1,14 +1,26 @@name: Verify recording (real capture)# Records a marker stimulus through Cap's real Studio and Instant pipelines-# (ScreenCaptureKit + VideoToolbox) on hosted macOS runners and checks every-# frame and audio marker against the host-clock truth. See+# (ScreenCaptureKit + VideoToolbox) on macOS runners and checks every frame+# and audio marker against the host-clock truth. See# crates/verify/tests/recording.rs.--# Pull requests (and pushes to the suite branches) that touch the capture,-# encode, timestamp or verify code run the FAST tier once on macos-15; the-# FULL tier runs on macos-15 and macos-26 nightly and on dispatch, and for a-# pull request that touches a high-risk region of scripts/verify/guards.toml.+#+# Every pull request gets a "Verify recording" verdict (the job to require):+# - Scope decides from the diff whether the change reaches the recorder's or+# the stimulus's dependency closure or the harness (scripts/verify/plan.py+# recording); a pull request that does not is green without running;+# - one that does runs the FAST tier on Blacksmith macOS 15, or the FULL tier+# on Blacksmith macOS 15 and 26 when it touches a high-risk region or an+# engine crate; nightly and dispatch run FULL on both.+#+# Runners: Blacksmith macOS (M4 Pro VM, 6 vCPU, logged-in GUI session, no+# concurrency cap). Hosted macOS is reserved for main and release CI and is+# only used on dispatch (runners=hosted or both). Blacksmith VMs have no+# VideoToolbox hardware H.264 encoder (probe run 38847791287), so recordings+# there take the software (libx264) path; the hardware path is covered by+# dispatches on hosted runners. Blacksmith needs Screen Recording and+# Microphone granted in TCC, its display woken, and an audio device+# (BlackHole); the setup step below does that.on:workflow_dispatch:inputs:@@ -25,47 +37,26 @@ on:description: "Preserve recordings and raw crash snapshots for passing cases"type: booleandefault: false+ system_audio:+ description: "Blacksmith only: play and require system audio through BlackHole (not yet characterised)"+ type: boolean+ default: false+ runners:+ description: "blacksmith (default), hosted (GitHub macOS, reserved for main and release) or both"+ type: choice+ options: [blacksmith, hosted, both]+ default: blacksmithschedule:- cron: "41 3 * * *"pull_request:- paths:- - "crates/recording/**"- - "crates/scap-*/**"- - "crates/enc-avfoundation/**"- - "crates/timestamp/**"- - "crates/camera*/**"- - "crates/media-info/**"- - "crates/verify/**"- - "crates/project/**"- - "Cargo.lock"- - ".github/workflows/verify-recording.yml"- - "crates/enc-ffmpeg/**"- - "crates/audio/**"- - "crates/editor/src/audio*.rs"- - "crates/rendering/src/decoder/**"- - "crates/video-decode/**"- - "apps/desktop-gpui/src/upload*"- - "apps/desktop-gpui/src/upload/**"- - "scripts/verify/guards.toml"-+ types: [opened, synchronize, reopened, ready_for_review]push:branches:- ci/verify-ci- ci/verify-recording- paths:- - "crates/recording/**"- - "crates/scap-*/**"- - "crates/enc-avfoundation/**"- - "crates/timestamp/**"- - "crates/camera*/**"- - "crates/media-info/**"- - "crates/verify/**"- - "crates/project/**"- - "Cargo.lock"- - ".github/workflows/verify-recording.yml"concurrency:- group: verify-recording-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name }}+ group: verify-recording-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && format('-{0}', github.run_id) || '' }}cancel-in-progress: ${{ github.event_name != 'schedule' }}permissions:@@ -78,42 +69,71 @@ jobs:runs-on: ubuntu-latesttimeout-minutes: 5outputs:+ run: ${{ steps.scope.outputs.run }}runners: ${{ steps.scope.outputs.runners }}tier: ${{ steps.scope.outputs.tier }}steps:- uses: actions/checkout@v4with:persist-credentials: false- sparse-checkout: scripts/verify+- name: Scopeid: scopeenv:GH_TOKEN: ${{ github.token }}EVENT: ${{ github.event_name }}PR: ${{ github.event.pull_request.number }}+ BEFORE: ${{ github.event.before }}INPUT_TIER: ${{ inputs.tier }}+ INPUT_RUNNERS: ${{ inputs.runners }}+ RUSTUP_TOOLCHAIN: stablerun: |set -euo pipefail- both='["macos-15","macos-26"]'+ files="$RUNNER_TEMP/changed.txt"+ blacksmith='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]'case "$EVENT" in- schedule) runners="$both"; tier=full ;;- workflow_dispatch) runners="$both"; tier="${INPUT_TIER:-full}" ;;- pull_request)- gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"- python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt"- if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt"; then- runners="$both"; tier=full+ schedule) run=true; runners="$blacksmith"; tier=full ;;+ workflow_dispatch)+ run=true+ tier="${INPUT_TIER:-full}"+ case "${INPUT_RUNNERS:-blacksmith}" in+ hosted) runners='["macos-15","macos-26"]' ;;+ both) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' ;;+ *) runners="$blacksmith" ;;+ esac ;;+ *)+ if [[ "$EVENT" == pull_request ]]; then+ gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$files"+ max=3000+ elif [[ "$BEFORE" =~ ^0+$ ]]; then+ : > "$files"; echo Cargo.toml >> "$files"+ max=300else- runners='["macos-15"]'; tier=fast+ gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$GITHUB_SHA" --jq '.files[].filename' > "$files"+ max=300+ fi+ python3 scripts/verify/plan.py recording --files "$files" --max-files "$max" > "$RUNNER_TEMP/scope.txt"+ cat "$RUNNER_TEMP/scope.txt"+ run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")"+ python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt"+ if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; then+ runners="$blacksmith"; tier=full+ else+ runners='["blacksmith-6vcpu-macos-15"]'; tier=fastfi ;;- *) runners='["macos-15"]'; tier=fast ;;esac+ echo "run=$run" >> "$GITHUB_OUTPUT"echo "runners=$runners" >> "$GITHUB_OUTPUT"echo "tier=$tier" >> "$GITHUB_OUTPUT"- echo "real capture: $tier tier on $runners" | tee -a "$GITHUB_STEP_SUMMARY"+ if [[ "$run" == true ]]; then+ echo "real capture: $tier tier on $runners" | tee -a "$GITHUB_STEP_SUMMARY"+ else+ echo "real capture: no recording code changed, nothing to run" | tee -a "$GITHUB_STEP_SUMMARY"+ fireal-capture:needs: scope+ if: needs.scope.outputs.run == 'true'strategy:fail-fast: falsematrix:@@ -130,7 +150,21 @@ jobs:CAP_VERIFY_SCRATCH: ${{ github.workspace }}/verify-scratchCAP_VERIFY_REPORT_DIR: ${{ github.workspace }}/verify-reportCAP_VERIFY_DUMP: ${{ github.workspace }}/verify-dump- CAP_VERIFY_AUDIO_OUTPUT: ${{ matrix.runner == 'macos-26' && 'Null Audio Device' || '' }}+ CAP_VERIFY_HOST: ${{ startsWith(matrix.runner, 'blacksmith-') && 'blacksmith' || 'github-hosted' }}+ # A case this runner cannot run fails, except system audio as the only+ # audio while the stimulus stays silent (see below).+ CAP_VERIFY_REQUIRE_SUPPORTED: "1"+ CAP_VERIFY_UNSUPPORTED_ALLOW: ${{ !(startsWith(matrix.runner, 'blacksmith-') && inputs.system_audio) && 'recording/instant/system-only' || '' }}+ # Hosted runners' output devices give ScreenCaptureKit only silence, so+ # system audio is not checked there. Blacksmith gets BlackHole (set up+ # below), which does carry the stimulus's audio on macOS 15 (not on+ # 26), but its system-audio timing is not characterised yet (run+ # 38843005408: system audio +31 to +44 ms against the mic, BlackHole+ # latency not in the truth), so by default the stimulus stays silent+ # there too; dispatch system_audio=true plays and requires it.+ CAP_VERIFY_AUDIO_OUTPUT: ${{ startsWith(matrix.runner, 'blacksmith-') && 'BlackHole 2ch' || matrix.runner == 'macos-26' && 'Null Audio Device' || '' }}+ CAP_VERIFY_STIMULUS_AUDIO: ${{ startsWith(matrix.runner, 'blacksmith-') && !inputs.system_audio && '0' || '' }}+ CAP_VERIFY_REQUIRE_SYSTEM_AUDIO: ${{ startsWith(matrix.runner, 'blacksmith-') && inputs.system_audio && '1' || '' }}steps:- uses: actions/checkout@v4@@ -174,6 +208,78 @@ jobs:cargo test --locked -p cap-verify --test recording --no-runecho "build seconds: $(( $(date +%s) - start ))"+ # Blacksmith runs a logged-in GUI session with SIP disabled, but:+ # - ScreenCaptureKit is denied until TCC grants the responsible process+ # (Runner.Worker), so Screen Recording and Microphone are granted to+ # every binary that captures plus the whole runner ancestry, in the+ # system and the user TCC database (Microphone is a per-user+ # service: without it CoreAudio blocks the stimulus forever in+ # AudioDeviceCreateIOProcID waiting for coreaudiod), then tccd+ # restarts;+ # - the display starts asleep (CGDisplayIsAsleep, no active display),+ # so ScreenCaptureKit lists windows but no display: wake it and keep+ # it awake for the job;+ # - there is no audio device: BlackHole provides input and output.+ - name: Blacksmith capture permissions and audio device+ if: startsWith(matrix.runner, 'blacksmith-')+ run: |+ set -euo pipefail+ sudo pmset -a displaysleep 0 sleep 0 disksleep 0+ caffeinate -u -t 2+ nohup caffeinate -dimsu > /dev/null 2>&1 &+ brew install --cask blackhole-2ch+ brew install switchaudio-osx+ sudo killall coreaudiod || true+ for i in $(seq 1 30); do+ SwitchAudioSource -a -t output | grep -qx "BlackHole 2ch" && break+ sleep 1+ done+ SwitchAudioSource -s "BlackHole 2ch" -t output+ SwitchAudioSource -s "BlackHole 2ch" -t input+ echo "outputs:"; SwitchAudioSource -a -t output+ echo "default output: $(SwitchAudioSource -c -t output)"++ { cargo build --locked -p cap-verify-stimulus --message-format=json 2>/dev/null; \+ cargo test --locked -p cap-verify --test recording --no-run --message-format=json 2>/dev/null; } \+ | python3 -c 'import json, sys+ for line in sys.stdin:+ m = json.loads(line)+ if m.get("reason") == "compiler-artifact" and m.get("executable") and m["target"]["name"] in ("recording", "cap-verify-stimulus"):+ print(m["executable"])' > "$RUNNER_TEMP/capture-binaries"+ test "$(wc -l < "$RUNNER_TEMP/capture-binaries")" -ge 2+ p=$$+ while [ "$p" -gt 1 ]; do+ ps -o comm= -p "$p"+ p=$(ps -o ppid= -p "$p" | tr -d ' ')+ done >> "$RUNNER_TEMP/capture-binaries"+ for c in "$(rustup which cargo)" "$HOME/.cargo/bin/cargo"; do echo "$c"; done >> "$RUNNER_TEMP/capture-binaries"++ db="/Library/Application Support/com.apple.TCC/TCC.db"+ user_db="$HOME/Library/Application Support/com.apple.TCC/TCC.db"+ approvals="$HOME/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist"+ mkdir -p "$(dirname "$approvals")"+ now=$(date +%s)+ until=$(date -u -v+1y '+%a %b %d %H:%M:%S GMT %Y')+ sort -u "$RUNNER_TEMP/capture-binaries" | grep '^/' | while read -r client; do+ for service in kTCCServiceScreenCapture kTCCServiceMicrophone; do+ row="INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);"+ sudo sqlite3 "$db" "$row"+ sqlite3 "$user_db" "$row"+ done+ /usr/libexec/PlistBuddy -c "Add :$client date $until" "$approvals" >/dev/null 2>&1 || true+ echo "granted $client"+ done+ killall tccd || true+ sudo killall tccd || true+ sleep 2+ cat > "$RUNNER_TEMP/display.swift" <<'SWIFT'+ import CoreGraphics+ let main = CGMainDisplayID()+ print("display \(main): asleep \(CGDisplayIsAsleep(main)), active \(CGDisplayIsActive(main)), \(CGDisplayBounds(main)), screen capture \(CGPreflightScreenCaptureAccess())")+ SWIFT+ swiftc "$RUNNER_TEMP/display.swift" -o "$RUNNER_TEMP/display"+ "$RUNNER_TEMP/display"+- name: Real-capture recording oraclerun: |mkdir -p "$CAP_VERIFY_DUMP"@@ -196,6 +302,11 @@ jobs:echo "test seconds: $(( $(date +%s) - start ))"exit "$status"+ - name: Summary+ if: always()+ run: |+ python3 scripts/verify/summary.py --title "real capture ${{ matrix.runner }} ${{ needs.scope.outputs.tier }}" "$CAP_VERIFY_REPORT_DIR" >> "$GITHUB_STEP_SUMMARY" || true+- name: Upload artifactsif: always()uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2@@ -206,3 +317,56 @@ jobs:${{ github.workspace }}/verify-dump${{ github.workspace }}/verify-scratchif-no-files-found: ignore++ # The check to require: green when the change is out of scope or every+ # planned runner passed with at least the tier's case count.+ gate:+ name: Verify recording+ needs: [scope, real-capture]+ if: always()+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ - uses: actions/checkout@v4+ with:+ persist-credentials: false+ sparse-checkout: scripts/verify++ - uses: actions/download-artifact@v4+ if: needs.scope.outputs.run == 'true'+ with:+ pattern: verify-recording-*+ path: ${{ runner.temp }}/reports++ - name: Verdict+ env:+ SCOPE: ${{ needs.scope.result }}+ RUN: ${{ needs.scope.outputs.run }}+ CAPTURE: ${{ needs.real-capture.result }}+ TIER: ${{ needs.scope.outputs.tier }}+ RUNNERS: ${{ needs.scope.outputs.runners }}+ CASE_FILTER: ${{ inputs.case_filter }}+ run: |+ if [[ "$SCOPE" != success ]]; then+ echo "scoping failed" >&2+ exit 1+ fi+ if [[ "$RUN" != true ]]; then+ echo "No recording code changed: nothing to verify." | tee -a "$GITHUB_STEP_SUMMARY"+ exit 0+ fi+ if [[ "$CAPTURE" != success ]]; then+ echo "real capture $CAPTURE ($TIER tier): see the runner jobs" >&2+ exit 1+ fi+ if [[ -n "$CASE_FILTER" ]]; then+ echo "case filter set: executed-case floor not applied"+ exit 0+ fi+ floor=0+ python3 scripts/verify/floor.py --recording --tier "$TIER" --runners "$RUNNERS" "$RUNNER_TEMP/reports" > "$RUNNER_TEMP/floor.txt" || floor=$?+ cat "$RUNNER_TEMP/floor.txt" | tee -a "$GITHUB_STEP_SUMMARY"+ if [[ $floor != 0 ]]; then+ echo "::error::fewer real-capture cases ran than planned"+ exit 1+ fi