:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Baseline resolution, signing, and committing the approved baseline back to GitHub."""
from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING
from uuid import UUID
from django.conf import settings
from django.db.models import F, Q
import structlog
from posthog.egress.limiter.policies import Priority
if TYPE_CHECKING:
from posthog.models.integration import GitHubIntegration
from ..db import WRITER_DB
from ..facade.enums import ReviewState, SnapshotResult
from ..models import Repo, Run, RunSnapshot
from ..signing import sign_snapshot_hash, verify_signed_hash
from . import errors, github_api, run_queries
logger = structlog.get_logger(__name__)
def _verify_baseline_hashes(repo: Repo, raw_hashes: dict[str, str]) -> dict[str, str]:
"""Verify HMAC signatures on baseline hashes from the CLI.
Accepts a dict of ``{identifier: signed_hash_string}``. Returns a
dict of ``{identifier: plain_content_hash}`` for entries with valid
signatures. Invalid or unsigned entries are silently dropped —
they'll be treated as having no baseline (result = NEW).
"""
if not raw_hashes:
return {}
keys = repo.signing_keys or {}
if not keys:
# Legitimate baseline files only exist after the server's approval flow
# has written one (which populates signing_keys). Reaching here means a
# .snapshots.yml was committed before any approval — likely hand-crafted.
# Drop every entry rather than passing it through unsigned. Snapshots
# will classify NEW, surfacing the situation to a reviewer.
logger.warning(
"visual_review.baseline_no_signing_keys",
repo_id=str(repo.id),
entry_count=len(raw_hashes),
)
return {}
repo_id = str(repo.id)
verified: dict[str, str] = {}
for identifier, signed_hash in raw_hashes.items():
content_hash = verify_signed_hash(repo_id, identifier, signed_hash, keys)
if content_hash is not None:
verified[identifier] = content_hash
else:
logger.debug(
"visual_review.baseline_hash_rejected",
identifier=identifier,
reason="invalid_signature",
)
return verified
def _resolve_baselines_at_ref(repo: Repo, github: GitHubIntegration, run_type: str, ref: str) -> dict[str, str]:
"""Fetch baseline content hashes from GitHub at a specific ref (branch name or SHA).
Returns a dict of identifier → content_hash (plain, not signed).
Returns empty dict when baseline file doesn't exist.
"""
baseline_paths = repo.baseline_file_paths or {}
baseline_path = baseline_paths.get(run_type) or baseline_paths.get("default", ".snapshots.yml")
baselines_signed, _sha = github_api._fetch_baseline_file(github, repo.repo_full_name, baseline_path, ref)
return _verify_baseline_hashes(
repo,
{
identifier: entry["hash"]
for identifier, entry in baselines_signed.items()
if isinstance(entry, dict) and "hash" in entry
},
)
def _run_is_on_default_branch(repo: Repo, branch: str) -> bool:
"""Whether this run targets the repo's GitHub default branch.
Fences the client-supplied ``is_partial`` flag: the default branch holds
the authoritative full baseline, so a partial run there must not skip
removed-baseline detection. Resolves the default branch server-side from
GitHub. Returns ``False`` when it can't be determined (no integration) —
harmless, since the baseline fetch then returns empty and removal
detection short-circuits regardless.
"""
try:
github = github_api.get_github_integration_for_repo(repo)
except Exception:
logger.info("visual_review.no_github_integration", repo_id=str(repo.id))
return False
return branch == github_api._get_default_branch(github, repo.repo_full_name)
def _resolve_baselines(repo, run_type: str, branch: str) -> dict[str, str]:
"""Fetch baseline content hashes from GitHub for snapshot comparison.
Returns a dict of identifier → content_hash (plain, not signed).
Returns empty dict when no GitHub integration exists or the baseline
file is missing (first run).
"""
try:
github = github_api.get_github_integration_for_repo(repo)
except Exception:
logger.info("visual_review.no_github_integration", repo_id=str(repo.id))
return {}
return _resolve_baselines_at_ref(repo, github, run_type, branch)
def _resolve_baselines_with_merge_base(
repo: Repo,
run_type: str,
branch: str,
*,
rendered_identifiers: set[str],
commit_sha: str | None = None,
) -> tuple[dict[str, str], int]:
"""Fetch branch baseline merged with merge-base baseline.
The branch baseline tracks approvals. The merge-base baseline
fills in entries that were lost during a rebase (the bot commit
rewrites the full file, and git rebase replays it destructively).
Branch entries win on conflict so approvals are preserved.
A merge-queue branch heals nothing, because the tested file is the one
that lands. An entry missing for a story the run did not render is one the
branch deleted on purpose, and restoring it only manufactures a REMOVED
that no one on a queue branch can approve, so it reds the batch and every
pull request sharing it until the deleting one lands. An entry missing for
a story that still renders would delete a live baseline on merge, so it
raises BaselineEntriesLostError and the run fails.
Ordinary branches keep healing everything, because there the REMOVED is
the review gate: it is how a reviewer is asked to confirm that a story
should go. The queue signal is the server-verified source PR rather than
the branch name, which is client-supplied — matching the name alone would
let any caller skip that gate. When verification fails the filter stays
off, so a GitHub blip costs a red batch rather than a silent removal.
Identifiers previously approved as REMOVED on this branch are
tombstoned — healing would otherwise resurrect them from master
and re-flag them as removed on every subsequent run.
When *commit_sha* is provided the baseline is fetched at that exact
commit rather than the branch tip. The tip moves under a concurrent
push, and an ephemeral merge-queue branch is deleted once its batch
resolves — a fetch by branch name then 404s, which is indistinguishable
from "no baseline file" and reports the whole suite as new.
Returns (merged_baseline, healed_count).
"""
try:
github = github_api.get_github_integration_for_repo(repo)
except Exception:
logger.info("visual_review.no_github_integration", repo_id=str(repo.id))
return {}, 0
default_branch = github_api._get_default_branch(github, repo.repo_full_name)
# Pin the baseline to the exact commit under test so back-to-back pushes
# don't race, and so a branch deleted mid-run still resolves.
baseline_ref = commit_sha or branch
branch_baseline = _resolve_baselines_at_ref(repo, github, run_type, baseline_ref)
if branch == default_branch:
return branch_baseline, 0
# Compare from the same ref the baseline was read at. A deleted branch 404s
# here, and healing would then switch off exactly when it is needed.
merge_base_sha = github_api._get_merge_base_sha(github, repo.repo_full_name, default_branch, baseline_ref)
if not merge_base_sha:
return branch_baseline, 0
try:
merge_base_baseline = _resolve_baselines_at_ref(repo, github, run_type, merge_base_sha)
except Exception:
logger.warning(
"visual_review.merge_base_baseline_fetch_failed",
repo_id=str(repo.id),
branch=branch,
merge_base_sha=merge_base_sha,
)
return branch_baseline, 0
if not merge_base_baseline:
return branch_baseline, 0
source_pr_number = github_api._verified_merge_queue_source_pr(
github, repo.repo_full_name, branch, head_ref=baseline_ref
)
tombstoned = _tombstoned_identifiers(repo, run_type, branch, source_pr_number=source_pr_number)
on_merge_queue_branch = source_pr_number is not None
healable_merge_base = {
identifier: baseline_hash
for identifier, baseline_hash in merge_base_baseline.items()
if identifier not in tombstoned and (not on_merge_queue_branch or identifier in rendered_identifiers)
}
healed = set(healable_merge_base) - set(branch_baseline)
merged = {**healable_merge_base, **branch_baseline}
# The name check only adds failures, so trusting an unverified name cannot skip a gate.
if on_merge_queue_branch or github_api._MERGE_QUEUE_BRANCH_RE.match(branch):
# Raw key sets, not `healed`: a tombstoned story that renders again still needs its baseline.
lost = (set(merge_base_baseline) - set(branch_baseline)) & rendered_identifiers
if lost:
raise errors.BaselineEntriesLostError(sorted(lost))
if healed or tombstoned:
logger.info(
"visual_review.baseline_healed",
repo_id=str(repo.id),
branch=branch,
healed_count=len(healed),
branch_count=len(branch_baseline),
merge_base_count=len(merge_base_baseline),
merged_count=len(merged),
tombstoned_count=len(tombstoned),
)
return merged, len(healed)
def _tombstoned_identifiers(repo: Repo, run_type: str, branch: str, source_pr_number: int | None = None) -> set[str]:
"""Identifiers whose latest approved outcome on this branch was REMOVED.
Healing pulls entries from merge-base back into the baseline when
they're missing from branch. Without tombstoning, an approved
removal keeps reappearing: the bot commit drops it from the branch
file, but the next run's merge-base fetch re-adds it and classifies
it REMOVED all over again.
Uses the most recent approved decision per identifier so that a
later re-addition (approved as NEW/CHANGED) clears the tombstone.
``source_pr_number`` widens the scope to that PR's runs — pass only a
server-verified value (see ``_verified_merge_queue_source_pr``), never
one parsed from client-supplied input alone.
"""
from django.db.models import OuterRef, Subquery
branch_scope = Q(run__branch=branch)
if source_pr_number is not None:
branch_scope |= Q(run__pr_number=source_pr_number)
latest_approved_run = (
RunSnapshot.objects.using(WRITER_DB)
.filter(
branch_scope,
run__repo=repo,
run__run_type=run_type,
run__approved=True,
review_state=ReviewState.APPROVED,
identifier=OuterRef("identifier"),
)
.order_by("-run__created_at")
.values("run__created_at")[:1]
)
return set(
RunSnapshot.objects.using(WRITER_DB)
.filter(
branch_scope,
run__repo=repo,
run__run_type=run_type,
run__approved=True,
review_state=ReviewState.APPROVED,
result=SnapshotResult.REMOVED,
)
.annotate(latest_approved_at=Subquery(latest_approved_run))
.filter(run__created_at=F("latest_approved_at"))
.values_list("identifier", flat=True)
.distinct()
)
def _approved_baseline_updates(snapshots: Iterable[RunSnapshot]) -> list[dict]:
"""The committed-baseline update set: approved changed/new snapshots, by approved hash.
Derived from DB state so the commit always reflects every approval regardless of how
many calls it took. Tolerated snapshots are excluded — toleration never updates the baseline.
Quarantined snapshots are committed when approved, because only a per-identifier approval
can mark one: bulk approve skips quarantined rows, so a flapping hash never lands by accident.
A quarantined story still renders on every run. If its baseline cannot move while it is
quarantined, a code change to the story leaves the entry stale, and the drift fails every
run on the day the quarantine is lifted or expires.
"""
return [
{"identifier": s.identifier, "new_hash": s.approved_hash}
for s in snapshots
if s.review_state == ReviewState.APPROVED and s.result in (SnapshotResult.NEW, SnapshotResult.CHANGED)
]
def _build_snapshots_yaml(
repo: Repo,
current_baselines: dict[str, dict],
updates: list[dict],
) -> str:
"""Build updated snapshots.yml with HMAC-signed hashes.
Each snapshot value is ``{hash: "v1.<kid>.<blake3hex>.<mac>"}``
where the MAC binds the hash to the repo and identifier.
*current_baselines* maps identifier to ``{hash: signed_hash_str}``.
*updates* is a list of ``{identifier, new_hash}`` where ``new_hash``
is a plain content hash — it gets signed here.
"""
import yaml
kid, secret_hex = repo.get_active_signing_key()
repo_id = str(repo.id)
merged = dict(current_baselines)
for update in updates:
identifier = update["identifier"]
content_hash = update["new_hash"]
signed = sign_snapshot_hash(repo_id, identifier, content_hash, secret_hex, kid)
merged[identifier] = {"hash": signed}
sorted_snapshots = dict(sorted(merged.items()))
data: dict = {
"version": 1,
"config": {
"api": settings.SITE_URL,
"team": str(repo.team_id),
"repo": str(repo.id),
},
"snapshots": sorted_snapshots,
}
return yaml.dump(data, default_flow_style=False, sort_keys=False, indent=4)
def _get_coauthor_trailer(user_id: int, installation_id: str) -> str | None:
"""Return a `Co-authored-by` trailer for the approver, if they have a personal
GitHub integration for the same installation. Returns None when no match exists.
"""
from posthog.models.user_integration import UserGitHubIntegration, UserIntegration
user_integration = UserIntegration.objects.filter(
user_id=user_id,
kind=UserIntegration.IntegrationKind.GITHUB,
integration_id=installation_id,
).first()
if user_integration is None:
return None
return UserGitHubIntegration(user_integration, source="visual_review", priority=Priority.NORMAL).coauthor_trailer
def _commit_baseline_to_github(
run: Run, repo: Repo, approved_snapshots: list[dict], approver_user_id: int | None = None
) -> dict:
"""
Commit updated baseline file to GitHub.
Raises:
GitHubIntegrationNotFoundError: No GitHub integration for team
PRSHAMismatchError: PR has newer commits than this run
BaselineFilePathNotConfiguredError: No baseline path for run type
GitHubCommitError: GitHub API error
"""
baseline_paths = repo.baseline_file_paths or {}
baseline_path = baseline_paths.get(run.run_type) or baseline_paths.get("default", ".snapshots.yml")
if not baseline_path:
raise errors.BaselineFilePathNotConfiguredError(f"No baseline file path configured for run type {run.run_type}")
github = github_api.get_github_integration_for_repo(repo)
if run.pr_number is None:
raise errors.GitHubCommitError("Cannot commit to GitHub: run has no associated PR number")
pr_info = github_api._get_pr_info(github, repo.repo_full_name, run.pr_number)
if pr_info["head_sha"] != run.commit_sha:
raise errors.PRSHAMismatchError(
f"PR has newer commits. Expected {run.commit_sha}, got {pr_info['head_sha']}. "
"Please re-run visual tests on the latest commit."
)
current_baselines, file_sha = github_api._fetch_baseline_file(
github, repo.repo_full_name, baseline_path, pr_info["head_ref"]
)
# Remove entries for snapshots detected as removed in this run
removed_identifiers = set(run.snapshots.filter(result=SnapshotResult.REMOVED).values_list("identifier", flat=True))
for identifier in removed_identifiers:
current_baselines.pop(identifier, None)
updates = [{"identifier": s["identifier"], "new_hash": s["new_hash"]} for s in approved_snapshots]
new_content = _build_snapshots_yaml(repo, current_baselines, updates)
# Use GitHubIntegration.update_file() - it expects just the repo name, not full path
# The org comes from github.organization()
repo_name = repo.repo_full_name.split("/")[-1] if "/" in repo.repo_full_name else repo.repo_full_name
updated_count = len(updates)
removed_count = len(removed_identifiers)
parts = [f"{updated_count} updated"]
if removed_count:
parts.append(f"{removed_count} removed")
summary = ", ".join(parts)
commit_message = f"chore(visual): update {run.run_type} baselines\n\n{summary}\nRun: {run.id}"
installation_id = github.integration.integration_id
if approver_user_id is not None and isinstance(installation_id, str) and installation_id:
trailer = _get_coauthor_trailer(approver_user_id, installation_id)
if trailer:
commit_message = f"{commit_message}\n\n{trailer}"
result = github.update_file(
repository=repo_name,
file_path=baseline_path,
content=new_content,
commit_message=commit_message,
branch=pr_info["head_ref"],
sha=file_sha,
)
if not result.get("success"):
raise errors.GitHubCommitError(f"Failed to commit baseline: {result.get('error')}")
run.metadata["baseline_commit_sha"] = result.get("commit_sha")
run.save(update_fields=["metadata"])
return result
def build_signed_baseline(run_id: UUID, team_id: int | None = None) -> str:
"""Build the signed baseline YAML for a finalized run, without committing it.
For callers that commit the baseline themselves (the CLI's auto-approve, which writes
the file and commits via git rather than the GitHub App). Mirrors the committed set:
the current baseline merged with the approved changed/new hashes, removed entries pruned.
"""
run = run_queries.get_run_with_snapshots(run_id, team_id=team_id)
repo = run.repo
baseline_paths = repo.baseline_file_paths or {}
baseline_path = baseline_paths.get(run.run_type) or baseline_paths.get("default", ".snapshots.yml")
github = github_api.get_github_integration_for_repo(repo)
current_baselines, _file_sha = github_api._fetch_baseline_file(
github, repo.repo_full_name, baseline_path, run.branch
)
snapshots = list(run.snapshots.all())
for s in snapshots:
if s.result == SnapshotResult.REMOVED:
current_baselines.pop(s.identifier, None)
return _build_snapshots_yaml(
repo, current_baselines=current_baselines, updates=_approved_baseline_updates(snapshots)
)