Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): verify changed stories in webkit before the queue (#112704)

Raúl Negrón-Otero1f4afeff8a7a
diff --git a/.github/scripts/verify-storybook-new-stories.sh b/.github/scripts/verify-storybook-new-stories.sh
index a6988b96ac67..d55d19e1fa5e 100755
--- a/.github/scripts/verify-storybook-new-stories.sh
+++ b/.github/scripts/verify-storybook-new-stories.sh
@@ -2,7 +2,7 @@
set -euo pipefail
# Verify changed Storybook story files are stable by re-running them multiple times.
-# Catches flaky snapshots before they land on master.
+# Catches flaky snapshots and webkit-only breaks before they land on master.
#
# Usage:
# verify-storybook-new-stories.sh <base_sha> [repeat_count]
@@ -73,7 +73,7 @@ elif [ "$file_count" -gt 10 ] && [ "$REPEAT_COUNT" -gt 2 ]; then
REPEAT_COUNT=2
fi
-echo "Verifying $file_count file(s) × $REPEAT_COUNT runs:"
+echo "Verifying $file_count file(s): $REPEAT_COUNT chromium run(s), then 1 webkit run:"
printf " %s\n" "${stories_to_verify[@]}"
# Build one escaped path regex per changed story file.
@@ -91,12 +91,34 @@ echo ""
echo "testPathPatterns: ${pattern_args[*]}"
echo ""
+declare -a failed_runs=()
+
+run_stories() {
+ local browser="$1" run_name="$2" snapshot_flag="$3"
+ echo "=== $run_name ==="
+
+ # Run test-storybook directly (tests a pre-built storybook dist served over http-server).
+ # pipefail is set at script level so tee preserves the exit code.
+ # --passWithNoTests: changed stories may live in a separate storybook that
+ # the main runner's testMatch doesn't cover. Those are verified by their own
+ # CI, so finding no matching tests here is not a failure.
+ # Keep STORYBOOK_SKIP_TAGS in sync with the visual-regression shards in ci-storybook.yml.
+ if STORYBOOK_SKIP_TAGS="test-skip,test-skip-${browser}" \
+ pnpm --filter=@posthog/storybook exec test-storybook \
+ "$snapshot_flag" --no-index-json --maxWorkers=1 \
+ --browsers "$browser" \
+ -- "${pattern_args[@]}" --passWithNoTests 2>&1 | tee "/tmp/storybook-verify-${run_name}.log"; then
+ echo "$run_name passed"
+ else
+ echo "$run_name failed"
+ failed_runs+=("$run_name")
+ fi
+ echo ""
+}
+
# Run the stories REPEAT_COUNT times. Each run does a full snapshot comparison.
# If any run fails, the story is flaky.
-failed_runs=0
for run in $(seq 1 "$REPEAT_COUNT"); do
- echo "=== Run $run/$REPEAT_COUNT ==="
-
# First run: --updateSnapshot to create baselines for new stories.
# Subsequent runs: --ci to verify the snapshot is stable.
if [ "$run" -eq 1 ]; then
@@ -105,34 +127,15 @@ for run in $(seq 1 "$REPEAT_COUNT"); do
snapshot_flag="--ci"
fi
- set +e
- # Run test-storybook directly (tests a pre-built storybook dist served over http-server).
- # pipefail is set at script level so tee preserves the exit code.
- # --passWithNoTests: changed stories may live in a separate storybook that
- # the main runner's testMatch doesn't cover. Those are verified by their own
- # CI, so finding no matching tests here is not a failure.
- pnpm --filter=@posthog/storybook exec test-storybook \
- $snapshot_flag --no-index-json --maxWorkers=1 \
- --browsers chromium \
- -- "${pattern_args[@]}" --passWithNoTests 2>&1 | tee "/tmp/storybook-verify-run${run}.log"
- exit_code=${PIPESTATUS[0]}
- set -e
-
- if [ $exit_code -ne 0 ]; then
- echo "Run $run failed (exit code $exit_code)"
- failed_runs=$((failed_runs + 1))
- else
- echo "Run $run passed"
- fi
-
- echo ""
+ run_stories chromium "chromium-run-$run" "$snapshot_flag"
done
-if [ "$failed_runs" -gt 0 ]; then
- echo ""
- echo "Flake verification failed — $failed_runs/$REPEAT_COUNT runs failed"
- echo "Flaky snapshots must be fixed before merging."
+# PR matrices render chromium only. --updateSnapshot stops a missing baseline from failing this run.
+run_stories webkit webkit-run --updateSnapshot
+
+if [ "${#failed_runs[@]}" -gt 0 ]; then
+ echo "Flake verification failed in: ${failed_runs[*]}. Fix the stories before merging."
exit 1
fi
-echo "Flake verification passed — all $REPEAT_COUNT runs stable"
+echo "Flake verification passed"
diff --git a/.github/workflows/ci-storybook.yml b/.github/workflows/ci-storybook.yml
index 96e46b5d7716..41c916e2eb7a 100644
--- a/.github/workflows/ci-storybook.yml
+++ b/.github/workflows/ci-storybook.yml
@@ -212,7 +212,7 @@ jobs:
# common/storybook/.storybook/test-runner.ts), so webkit adds render-error coverage
# only. That coverage is worth a batched queue run, because a webkit-only break then
# fails the queue instead of master. It is not worth every push of every frontend PR,
- # so author PRs stay chromium-only.
+ # so author PRs stay chromium-only. Flake verification still renders changed stories in webkit.
- name: Build matrix
id: matrix
env:
@@ -918,13 +918,13 @@ jobs:
run: exit 1
# Verify changed story files are stable by re-running them multiple times.
- # Catches flaky snapshots before they land on master.
+ # Catches flaky snapshots and webkit-only breaks before they land on master.
flake-verification:
name: Storybook flake verification
runs-on: ubuntu-24.04
needs: [changes, build-storybook]
# Skipped on trunk-merge/**: the job diffs against the merge base to find changed stories,
- # which on a queue branch is the same set the PR's own run already re-ran three times.
+ # which on a queue branch is the same set the PR's own run already verified.
if: |
needs.changes.outputs.frontend == 'true'
&& github.event_name == 'pull_request'
@@ -1036,6 +1036,7 @@ jobs:
build-storybook,
select-stories,
visual-regression,
+ flake-verification,
vr-setup,
vr-complete,
check-baseline-removals,
@@ -1076,6 +1077,11 @@ jobs:
exit 1
fi
+ if [[ "${{ needs.flake-verification.result }}" != "success" && "${{ needs.flake-verification.result }}" != "skipped" ]]; then
+ echo "Storybook flake verification did not pass (result: ${{ needs.flake-verification.result }})."
+ exit 1
+ fi
+
if [[ "${{ needs.vr-setup.result }}" != "success" && "${{ needs.vr-setup.result }}" != "skipped" ]]; then
echo "Visual Review setup failed or was cancelled (result: ${{ needs.vr-setup.result }})."
exit 1