:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): verify changed stories in webkit before the queue (#112704)
Raúl Negrón-Otero1f4afeff8a7a
diff --git a/.github/scripts/verify-storybook-new-stories.sh b/.github/scripts/verify-storybook-new-stories.shindex a6988b96ac67..d55d19e1fa5e 100755--- a/.github/scripts/verify-storybook-new-stories.sh+++ b/.github/scripts/verify-storybook-new-stories.sh@@ -2,7 +2,7 @@set -euo pipefail# Verify changed Storybook story files are stable by re-running them multiple times.-# Catches flaky snapshots before they land on master.+# Catches flaky snapshots and webkit-only breaks before they land on master.## Usage:# verify-storybook-new-stories.sh <base_sha> [repeat_count]@@ -73,7 +73,7 @@ elif [ "$file_count" -gt 10 ] && [ "$REPEAT_COUNT" -gt 2 ]; thenREPEAT_COUNT=2fi-echo "Verifying $file_count file(s) × $REPEAT_COUNT runs:"+echo "Verifying $file_count file(s): $REPEAT_COUNT chromium run(s), then 1 webkit run:"printf " %s\n" "${stories_to_verify[@]}"# Build one escaped path regex per changed story file.@@ -91,12 +91,34 @@ echo ""echo "testPathPatterns: ${pattern_args[*]}"echo ""+declare -a failed_runs=()++run_stories() {+ local browser="$1" run_name="$2" snapshot_flag="$3"+ echo "=== $run_name ==="++ # Run test-storybook directly (tests a pre-built storybook dist served over http-server).+ # pipefail is set at script level so tee preserves the exit code.+ # --passWithNoTests: changed stories may live in a separate storybook that+ # the main runner's testMatch doesn't cover. Those are verified by their own+ # CI, so finding no matching tests here is not a failure.+ # Keep STORYBOOK_SKIP_TAGS in sync with the visual-regression shards in ci-storybook.yml.+ if STORYBOOK_SKIP_TAGS="test-skip,test-skip-${browser}" \+ pnpm --filter=@posthog/storybook exec test-storybook \+ "$snapshot_flag" --no-index-json --maxWorkers=1 \+ --browsers "$browser" \+ -- "${pattern_args[@]}" --passWithNoTests 2>&1 | tee "/tmp/storybook-verify-${run_name}.log"; then+ echo "$run_name passed"+ else+ echo "$run_name failed"+ failed_runs+=("$run_name")+ fi+ echo ""+}+# Run the stories REPEAT_COUNT times. Each run does a full snapshot comparison.# If any run fails, the story is flaky.-failed_runs=0for run in $(seq 1 "$REPEAT_COUNT"); do- echo "=== Run $run/$REPEAT_COUNT ==="-# First run: --updateSnapshot to create baselines for new stories.# Subsequent runs: --ci to verify the snapshot is stable.if [ "$run" -eq 1 ]; then@@ -105,34 +127,15 @@ for run in $(seq 1 "$REPEAT_COUNT"); dosnapshot_flag="--ci"fi- set +e- # Run test-storybook directly (tests a pre-built storybook dist served over http-server).- # pipefail is set at script level so tee preserves the exit code.- # --passWithNoTests: changed stories may live in a separate storybook that- # the main runner's testMatch doesn't cover. Those are verified by their own- # CI, so finding no matching tests here is not a failure.- pnpm --filter=@posthog/storybook exec test-storybook \- $snapshot_flag --no-index-json --maxWorkers=1 \- --browsers chromium \- -- "${pattern_args[@]}" --passWithNoTests 2>&1 | tee "/tmp/storybook-verify-run${run}.log"- exit_code=${PIPESTATUS[0]}- set -e-- if [ $exit_code -ne 0 ]; then- echo "Run $run failed (exit code $exit_code)"- failed_runs=$((failed_runs + 1))- else- echo "Run $run passed"- fi-- echo ""+ run_stories chromium "chromium-run-$run" "$snapshot_flag"done-if [ "$failed_runs" -gt 0 ]; then- echo ""- echo "Flake verification failed — $failed_runs/$REPEAT_COUNT runs failed"- echo "Flaky snapshots must be fixed before merging."+# PR matrices render chromium only. --updateSnapshot stops a missing baseline from failing this run.+run_stories webkit webkit-run --updateSnapshot++if [ "${#failed_runs[@]}" -gt 0 ]; then+ echo "Flake verification failed in: ${failed_runs[*]}. Fix the stories before merging."exit 1fi-echo "Flake verification passed — all $REPEAT_COUNT runs stable"+echo "Flake verification passed"diff --git a/.github/workflows/ci-storybook.yml b/.github/workflows/ci-storybook.ymlindex 96e46b5d7716..41c916e2eb7a 100644--- a/.github/workflows/ci-storybook.yml+++ b/.github/workflows/ci-storybook.yml@@ -212,7 +212,7 @@ jobs:# common/storybook/.storybook/test-runner.ts), so webkit adds render-error coverage# only. That coverage is worth a batched queue run, because a webkit-only break then# fails the queue instead of master. It is not worth every push of every frontend PR,- # so author PRs stay chromium-only.+ # so author PRs stay chromium-only. Flake verification still renders changed stories in webkit.- name: Build matrixid: matrixenv:@@ -918,13 +918,13 @@ jobs:run: exit 1# Verify changed story files are stable by re-running them multiple times.- # Catches flaky snapshots before they land on master.+ # Catches flaky snapshots and webkit-only breaks before they land on master.flake-verification:name: Storybook flake verificationruns-on: ubuntu-24.04needs: [changes, build-storybook]# Skipped on trunk-merge/**: the job diffs against the merge base to find changed stories,- # which on a queue branch is the same set the PR's own run already re-ran three times.+ # which on a queue branch is the same set the PR's own run already verified.if: |needs.changes.outputs.frontend == 'true'&& github.event_name == 'pull_request'@@ -1036,6 +1036,7 @@ jobs:build-storybook,select-stories,visual-regression,+ flake-verification,vr-setup,vr-complete,check-baseline-removals,@@ -1076,6 +1077,11 @@ jobs:exit 1fi+ if [[ "${{ needs.flake-verification.result }}" != "success" && "${{ needs.flake-verification.result }}" != "skipped" ]]; then+ echo "Storybook flake verification did not pass (result: ${{ needs.flake-verification.result }})."+ exit 1+ fi+if [[ "${{ needs.vr-setup.result }}" != "success" && "${{ needs.vr-setup.result }}" != "skipped" ]]; thenecho "Visual Review setup failed or was cancelled (result: ${{ needs.vr-setup.result }})."exit 1