:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): delete the github backend workflow and router (#116513)
Raúl Negrón-Oteroe1ff77b5da2f
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.ymlindex 8b035741fdda..d29ba2293cd2 100644--- a/.depot/workflows/ci-backend.yml+++ b/.depot/workflows/ci-backend.yml@@ -1,16 +1,11 @@-# Depot CI copy of .github/workflows/ci-backend.yml. Lives under .depot/ so-# GitHub Actions ignores it; only Depot CI parses this file. Depot CI runs every-# pull request event here, and the gate job's own check is the required one.+# Backend CI. Lives under .depot/ so GitHub Actions ignores it; only Depot CI parses this+# file. Every pull request event, master push, manual run and the hourly master run start+# here, and the gate job's `Test results` check is the required one.#-# Apples-to-apples is the whole point — keep action pins, env vars, runner-# sizes, and pytest flags in sync with the canonical workflow. A drift-check-# job in .github/workflows/ci-backend-shadow-drift.yml fails when canonical-# changes without a matching depot bump.-#-# Intentional deltas vs canonical (everything else is kept apples-to-apples):-# - Every pull request event runs here. Canonical runs no backend tests on pull requests;-# its `Django Tests Pass` job only points at this workflow, and it has no push trigger.-# Depot's own checks carry no output, so the gate posts a `Test results` check with the+# This file began as a copy of a GitHub Actions workflow, and the local actions it uses are+# mirrors of .github/actions (.github/workflows/ci-depot-action-mirrors.yml keeps those in+# sync). The notes below record where it differs from the GitHub Actions shape:+# - Depot's own checks carry no output, so the gate posts a `Test results` check with the# failure summary.# - runs-on: depot-* labels instead of GitHub-hosted ubuntu-24.04# - turbo-discover adds the `origin` remote before its history fetch when it is missing.@@ -125,8 +120,6 @@ name: Backend CI on Depot# Depot CI copy of canonical Backend CI. Branch protection requires the gate job's# `Backend CI on Depot / Test results` check, posted by the PostHog tests app.# Pull requests, master pushes, manual runs and the hourly master run all start here.-# The drift check (.github/workflows/ci-backend-shadow-drift.yml) keeps this file in sync-# with canonical; it runs whenever either file changes.on:push:branches:@@ -1374,7 +1367,7 @@ jobs:fioverall=0# A checkout without the matching snapshots must use legacy mode.- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: product-poe-default" .github/workflows/ci-backend.yml; then+ if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: product-poe-default" .depot/workflows/ci-backend.yml; thenecho "::notice::Checked-out revision predates the person-on-events default, running product tests in legacy mode"export PERSON_ON_EVENTS_V2_ENABLED=falsefi@@ -3578,7 +3571,7 @@ jobs:COV_ARGS=""fi# A checkout without the matching test pins must use legacy mode.- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: person-on-events-default" .github/workflows/ci-backend.yml; then+ if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: person-on-events-default" .depot/workflows/ci-backend.yml; thenecho "::notice::Checked-out revision predates the person-on-events default, running Core in legacy mode"export PERSON_ON_EVENTS_V2_ENABLED=falsefidiff --git a/.github/scripts/ci_backend_depot_failures.py b/.github/scripts/ci_backend_depot_failures.pydeleted file mode 100644index 1a75a64a3cf1..000000000000--- a/.github/scripts/ci_backend_depot_failures.py+++ /dev/null@@ -1,46 +0,0 @@-"""Explain why a Depot backend workflow failed: each failed job's step and its log lines.--Depot's own checks carry no output, and its run pages need a Depot login. The GitHub-check a PR author opens prints these lines instead.-"""--import re-import json-import subprocess-from typing import Any--RUNNER_MARKER = re.compile(r"^##\[\w+\]")-# Depot keeps up to 8,000 characters per log line. A shorter cut keeps one long line-# from filling the annotation.-MAX_LINE = 500---def message(attempt: dict[str, Any]) -> str:- step = " ".join(f"{attempt['job_display_name']}: {attempt['error_message']}".split())- lines = (RUNNER_MARKER.sub("", line["content"])[:MAX_LINE] for line in attempt["relevant_lines"])- return "\n".join((step, *lines))---def escape_annotation(text: str) -> str:- return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")---def depot_ci(*args: str) -> Any:- result = subprocess.run(- ["depot", "ci", *args, "--output", "json"], capture_output=True, text=True, check=True, timeout=60- )- return json.loads(result.stdout)---def explain(org: str, workflow: str) -> list[str]:- """Workflow commands: one error annotation per failed step, with its log lines as the message.-- The log lines come from pull request code. Escaped as annotation data, the way- @actions/core escapes it, they stay on the annotation's line and cannot start a command.- """- try:- diagnosis = depot_ci("diagnose", "--workflow", workflow, "--org", org)- messages = [message(attempt) for attempt in diagnosis["representative_attempts"]]- except Exception as error:- return [f"::warning::Cannot read the Depot failure details: {type(error).__name__}"]- return [f"::error title=Failed on Depot::{escape_annotation(text)}" for text in messages]diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.pydeleted file mode 100644index 99c864d713b1..000000000000--- a/.github/scripts/ci_backend_relay.py+++ /dev/null@@ -1,575 +0,0 @@-#!/usr/bin/env python3-"""Follow the Depot CI run that took over one pull request event, and read one of its checks.--GitHub Actions hands a pull request event to Depot CI (see ci_backend_route.py), and Depot-starts a workflow for every pull request event. One commit can therefore carry Depot runs of-earlier events, a duplicate run of the same event, and runs of another pull request with the-same head. Depot check times and pull request lists cannot tell them apart:--- When Depot cancels a superseded run, it posts a cancelled check for each job that had not- started, with `started_at` set to the cancel time.-- Depot's checks sit in its app's check suite for the commit, which carries the branch of the- commit's first push. When that branch is not the pull request's head, the checks list no- pull request.--.agents/skills/depot-ci/references/posthog-check-run-semantics.md has the measurements.-So the Depot wait job's check name carries the event: the pull request number and its-`updated_at`, which both engines read from the same event payload. That check identifies the-event's run, and every other check of the run is matched by the Depot workflow id in its-details URL.--Two events of one commit that arrive within a second or two race the concurrency cancel of-both engines, and each engine can keep a different event. GitHub Actions alone can also keep-either event of such a pair, so when this event has no live Depot run, the relay follows the-run that Depot kept for the racing event.--Standard library only: the relay job runs this with the runner's python3 before any install.-"""--import os-import re-import sys-import json-import time-import subprocess-import http.client-import urllib.error-import urllib.parse-import urllib.request-from collections.abc import Callable, Iterable, Sequence-from dataclasses import dataclass-from datetime import datetime, timedelta-from enum import Enum-from typing import Any, Protocol--from ci_backend_depot_failures import depot_ci, explain--DEPOT_APP_ID = 219785-DEPOT_ORG = "ntsdt08fpt"-# The PostHog tests GitHub App. Depot's wait and gate jobs post their verdicts with it too, because-# Depot posts its own checks from a budget that runs out at peak and then delivers them late.-MIRROR_APP_ID = 2492437-# The Trunk merge queue tests each batch through a draft pull request on this branch.-MERGE_QUEUE_PREFIX = "trunk-merge/"---def is_merge_queue(head_ref: str) -> bool:- return head_ref.startswith(MERGE_QUEUE_PREFIX)---DEPOT_WORKFLOW = "Backend CI on Depot"-WAIT_JOB = "Wait for GitHub Actions to hand off backend tests"-# Renders the same text as the wait job's name expression in .depot/workflows/ci-backend.yml.-EVENT_SUFFIX = " (PR {pr}, event {event_at})"-# How a pull request event payload renders `updated_at`.-EVENT_TIME = "%Y-%m-%dT%H:%M:%SZ"-RACING_EVENT_SECONDS = 2-GATE_CHECK = f"{DEPOT_WORKFLOW} / Django Tests Pass on Depot"-# The mirror posts the gate under its own name, so that a pull request does not show two rows with one name.-MIRRORED_GATE_CHECK = f"{DEPOT_WORKFLOW} / Test results"-# Every name the mirror posts a check under. A Depot run on a workflow revision without the-# mirror's gate name posts Depot's name from the mirror.-MIRROR_NAMES = {GATE_CHECK: (GATE_CHECK, MIRRORED_GATE_CHECK)}-DEPOT_RUN_URL = re.compile(r"^https://depot\.dev/orgs/([^/?]+)/workflows/([a-z0-9]+)(?:[?/]|$)")-PENDING_STATES = frozenset({"queued", "in_progress", "pending", "waiting", "requested"})-CONCLUSIONS = frozenset(- {- "success",- "failure",- "cancelled",- "skipped",- "timed_out",- "neutral",- "action_required",- "stale",- "startup_failure",- }-)-API_ROOT = "https://api.github.com"-PAGE_SIZE = 100-# A 403 also covers a secondary rate limit, which clears, so a few refusals in a row are tolerated.-MAX_REFUSALS = 5-PREREQUISITES = ("Repo checks (depot-ubuntu-24.04)", "Validate OpenAPI types")-# Depot's key for the `django_tests` job of .depot/workflows/ci-backend.yml, and how it names job states.-GATE_JOB_KEY = "ci-backend.yml:django_tests"-DEPOT_LIVE_STATES = frozenset({"queued", "waiting", "running"})-DEPOT_JOB_CONCLUSIONS = {"finished": "success", "failed": "failure"}-# Under the `timeout-minutes` of the `Django Tests Pass` job, so the relay reports before GitHub ends the job.-GATE_DEADLINE_MINUTES = 90---class ReadRefusedError(RuntimeError):- """The check-runs API keeps refusing the token, so no verdict can be read."""---class ReadFailedError(RuntimeError):- """One read of the check-runs API failed, so this poll cannot tell which checks exist."""---@dataclass(frozen=True)-class CheckRun:- id: int- # The conclusion once the check completed, its status before that.- state: str- details_url: str- app_id: int = DEPOT_APP_ID- # The job attempt in a mirrored check's URL. Depot's own checks carry none.- attempt: str = ""-- @classmethod- def from_api(cls, run: dict[str, Any]) -> "CheckRun":- state = (run.get("conclusion") if run.get("status") == "completed" else run.get("status")) or ""- url = str(run.get("details_url") or "")- parsed = urllib.parse.urlparse(url)- query = urllib.parse.parse_qs(parsed.query)- match = re.fullmatch(rf"/orgs/{re.escape(DEPOT_ORG)}/workflows/([a-z0-9]+)", parsed.path)- attempt = ""- if parsed.scheme == "https" and parsed.netloc == "depot.dev" and match:- job = query.get("job", [""])[0]- url = f"https://depot.dev/orgs/{DEPOT_ORG}/workflows/{match[1]}"- if re.fullmatch(r"[a-z0-9]+", job):- url += f"?job={job}"- attempt = query.get("attempt", [""])[0]- else:- url = ""- return cls(- id=int(run["id"]),- state=state if state in PENDING_STATES | CONCLUSIONS else "unknown",- details_url=url,- app_id=int(run["app"]["id"]),- attempt=attempt if re.fullmatch(r"[a-z0-9]+", attempt) else "",- )-- @property- def depot_workflow(self) -> str | None:- match = DEPOT_RUN_URL.match(self.details_url)- return match.group(2) if match and match.group(1) == DEPOT_ORG else None---class Phase(Enum):- ABSENT = "absent"- STARTING = "starting"- DECLINED = "declined"- CANCELLED = "cancelled"- RUNNING = "running"- FINISHED = "finished"---@dataclass(frozen=True)-class Progress:- phase: Phase- # The state of the check behind the phase, empty when there is none.- state: str = ""- details_url: str = ""- root_failure: str = ""- root_check_id: int = 0---def event_check_name(job: str, pr_number: int, event_at: str) -> str:- return f"{DEPOT_WORKFLOW} / {job}{EVENT_SUFFIX.format(pr=pr_number, event_at=event_at)}"---def wait_check_name(pr_number: int, event_at: str) -> str:- return event_check_name(WAIT_JOB, pr_number, event_at)---def current_check(checks: Iterable[CheckRun], workflow: str | None) -> CheckRun | None:- """The check of the newest attempt of one job in `workflow`, from either app that posts it.-- Depot posts one check per job attempt but can deliver it late, so its check ids do not order- its checks against the mirror's. The mirror posts one check per attempt that ran, in order,- but a post can fail. The mirror decides while it has posted at least as many attempts as- Depot shows. Otherwise the mirror missed an attempt, and Depot's newest check decides.- """- own = [check for check in checks if workflow is not None and check.depot_workflow == workflow]- mirrored = [check for check in own if check.app_id == MIRROR_APP_ID]- native = [check for check in own if check.app_id != MIRROR_APP_ID]- if mirrored and len(native) <= len({check.attempt or str(check.id) for check in mirrored}):- return max(mirrored, key=lambda check: check.id)- return max(native, key=lambda check: check.id, default=None)---def newest_live(runs: Sequence[CheckRun]) -> CheckRun | None:- """The newest run that was not cancelled, or the newest cancelled run when every run was."""- current = [check for workflow in {run.depot_workflow for run in runs} if (check := current_check(runs, workflow))]- live = [run for run in current if run.state != "cancelled"]- return max(live or current, key=lambda run: run.id, default=None)---def progress(wait: CheckRun | None, checks: Iterable[CheckRun]) -> Progress:- """Where the Depot run behind `wait` stands, judged by its check among `checks`."""- if wait is None:- return Progress(Phase.ABSENT)- if wait.state == "cancelled":- return Progress(Phase.CANCELLED, wait.state, wait.details_url)- if wait.state in PENDING_STATES:- return Progress(Phase.STARTING, wait.state, wait.details_url)- if wait.state != "success":- return Progress(Phase.DECLINED, wait.state, wait.details_url)- check = current_check(checks, wait.depot_workflow)- if check is None or check.state in PENDING_STATES:- return Progress(Phase.RUNNING, check.state if check else "", wait.details_url)- if check.state == "cancelled":- return Progress(Phase.CANCELLED, check.state, check.details_url)- return Progress(Phase.FINISHED, check.state, check.details_url)---class CheckReader(Protocol):- def read(self, name: str) -> list[CheckRun]: ...---class CheckRunReader:- """Reads one commit's Depot check runs by name from each app that posts them, with conditional requests.-- A 304 answer is free against the rate limit, so polling stays cheap.- """-- def __init__(- self,- repo: str,- sha: str,- token: str,- opener: Callable[..., Any] = urllib.request.urlopen,- pr_number: int | None = None,- app_ids: Sequence[int] = (MIRROR_APP_ID, DEPOT_APP_ID),- ) -> None:- self._repo = repo- self._sha = sha- self._token = token- self._pr_number = pr_number- self._opener = opener- self._app_ids = app_ids- self._cache: dict[tuple[str, int], tuple[str, list[CheckRun]]] = {}- self._refusals = 0-- def _url(self, name: str, app_id: int, page: int) -> str:- query = urllib.parse.urlencode(- {"check_name": name, "app_id": app_id, "filter": "all", "per_page": PAGE_SIZE, "page": page}- )- return f"{API_ROOT}/repos/{self._repo}/commits/{self._sha}/check-runs?{query}"-- def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]:- headers = {- "Accept": "application/vnd.github+json",- "X-GitHub-Api-Version": "2022-11-28",- "Authorization": f"Bearer {self._token}",- }- if etag:- headers["If-None-Match"] = etag- try:- with self._opener(urllib.request.Request(url, headers=headers), timeout=30) as response:- raw = response.read(2_000_001)- if len(raw) > 2_000_000:- raise ValueError("oversized check response")- return response.status, response.headers.get("ETag", ""), json.loads(raw.decode("utf-8"))- except urllib.error.HTTPError as error:- return error.code, "", {}-- def read(self, name: str) -> list[CheckRun]:- """Every app's checks of `name`, under each name the app posts it with.-- `current_check` picks the current one per workflow.- A failed read of any app raises, because the other app's checks alone can hold a stale attempt.- """- runs: list[CheckRun] = []- for app_id in self._app_ids:- app_names = MIRROR_NAMES.get(name, (name,)) if app_id == MIRROR_APP_ID else (name,)- for app_name in app_names:- app_runs = self._read_app(app_name, app_id)- if app_runs is None:- raise ReadFailedError(f"Cannot read {app_name}")- runs.extend(app_runs)- return runs-- def _read_app(self, name: str, app_id: int) -> list[CheckRun] | None:- """Read every page, reusing a cached answer only when the API confirms it with 304."""- etag, cached = self._cache.get((name, app_id), ("", []))- raw: list[dict[str, Any]] = []- page = 1- new_etag = ""- try:- while True:- code, page_etag, body = self._get(self._url(name, app_id, page), etag if page == 1 else "")- if page == 1 and code == 304:- self._refusals = 0- return cached- if code in (401, 403):- self._refusals += 1- if self._refusals >= MAX_REFUSALS:- raise ReadRefusedError(f"Cannot read checks for {self._sha}")- if code != 200:- sys.stdout.write(f"::warning::check-runs API returned {code}\n")- return None- self._refusals = 0- if page == 1:- new_etag = page_etag- batch = body["check_runs"]- if not isinstance(batch, list) or page > 20:- raise ValueError("invalid or excessive check pages")- raw.extend(batch)- if len(batch) < PAGE_SIZE:- break- page += 1- runs = [- CheckRun.from_api(run)- for run in raw- # One malformed record is skipped rather than discarding the whole answer.- if isinstance(run, dict)- and (run.get("app") or {}).get("id") == app_id- and run.get("name") == name- and run.get("head_sha") == self._sha- and (- not run.get("pull_requests")- or self._pr_number is None- or any(pr.get("number") == self._pr_number for pr in run["pull_requests"])- )- ]- runs = [run for run in runs if run.depot_workflow is not None]- except (OSError, http.client.HTTPException, KeyError, TypeError, ValueError, AttributeError):- sys.stdout.write("::warning::check-runs API read failed\n")- return None- # One page's ETag cannot validate the other pages of a paginated response.- self._cache[(name, app_id)] = (new_etag if page == 1 else "", runs)- return runs---@dataclass(frozen=True)-class Event:- repo: str- sha: str- pr_number: int- # The pull request's `updated_at` in this event's payload.- event_at: str- merge_queue: bool = False---def racing_wait(reader: CheckReader, event: Event, followed: set[str]) -> str | None:- """The newest racing wait check whose job took the hand-off or is pending, skipping `followed`."""- event_at = datetime.strptime(event.event_at, EVENT_TIME)- for offset in range(RACING_EVENT_SECONDS, -RACING_EVENT_SECONDS - 1, -1):- name = wait_check_name(event.pr_number, (event_at + timedelta(seconds=offset)).strftime(EVENT_TIME))- if name in followed:- continue- wait = newest_live(reader.read(name))- if progress(wait, ()).phase in (Phase.STARTING, Phase.RUNNING):- return name- return None---def prerequisite_failure(reader: CheckReader, wait: CheckRun, current: Progress) -> Progress:- for name in PREREQUISITES:- latest = current_check(reader.read(f"{DEPOT_WORKFLOW} / {name}"), wait.depot_workflow)- if latest and latest.state == "failure":- return Progress(Phase.FINISHED, "failure", current.details_url, name, latest.id)- return current---def poll(- reader: CheckReader,- event: Event,- check_name: str,- *,- deadline_minutes: int,- absent_minutes: int,- clock: Callable[[], float] = time.monotonic,- sleep: Callable[[float], None] = time.sleep,-) -> Progress:- """Polls until the event's check finishes, Depot declines the hand-off, or a deadline passes.-- A duplicate run of the same event can replace a cancelled one, so an absent or cancelled- run gets `absent_minutes` of grace. After the grace period, the run of a racing event- stands in for an absent or cancelled one.- """- start = clock()- event_name = wait_check_name(event.pr_number, event.event_at)- followed = {event_name}- current = Progress(Phase.ABSENT)- while True:- try:- wait = newest_live(reader.read(event_name))- checks = reader.read(check_name) if wait and wait.state == "success" else []- current = progress(wait, checks)- # Depot cancels its own run only after a deterministic prerequisite failure. A gate that- # failed without the cancel can follow a retryable one, so it keeps the retry options.- if current.phase == Phase.CANCELLED and wait and wait.state == "success":- current = prerequisite_failure(reader, wait, current)- sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n")- elapsed = clock() - start- if current.phase in (Phase.FINISHED, Phase.DECLINED):- return current- if current.phase in (Phase.ABSENT, Phase.CANCELLED) and elapsed >= absent_minutes * 60:- racing = racing_wait(reader, event, followed)- if racing is None:- return current- sys.stdout.write(f"Depot kept a racing event of this commit instead. Following: {racing}\n")- followed.add(racing)- event_name = racing- continue- except ReadFailedError as error:- # A failed read says nothing about the run, so it must not end the wait as absent.- sys.stdout.write(f"::warning::{error}. Reading again.\n")- elapsed = clock() - start- if elapsed >= deadline_minutes * 60:- return current- # The checked job only posts its check when the matrix is done, so once Depot has- # started, a slower poll costs at most a minute of latency.- sleep(60 if current.phase == Phase.RUNNING else 30)---def retry_failed_jobs(- org: str,- workflow: str,- *,- depot: Callable[..., Any] = depot_ci,- clock: Callable[[], float] = time.monotonic,- sleep: Callable[[float], None] = time.sleep,-) -> Progress | None:- """Retries the workflow's failed jobs on Depot and returns the gate's new verdict, or None when Depot cannot be reached.-- Depot refuses a retry while the workflow runs, so the retry waits for the workflow to finish.- A retry adds one execution to the workflow and queues the gate job in the same call. So the gate's- state is the new verdict once the workflow has more executions than it had when the retry was sent.- A gate that another retry turned green in the meantime is relayed as it is.- """- start = clock()- url = f"https://depot.dev/orgs/{org}/workflows/{workflow}"- executions_at_retry: int | None = None- failures = 0- while True:- sending = False- try:- shown = depot("workflow", "show", workflow, "--org", org)- gate = next(job["status"] for job in shown["jobs"] if job["job_key"] == GATE_JOB_KEY)- executions = len(shown["executions"])- status = shown["workflow"]["status"]- sys.stdout.write(f"Depot workflow: {status}, gate {gate}\n")- retried = executions_at_retry is not None and executions > executions_at_retry- if gate == "finished" or (retried and gate not in DEPOT_LIVE_STATES):- phase = Phase.CANCELLED if gate == "cancelled" else Phase.FINISHED- return Progress(phase, DEPOT_JOB_CONCLUSIONS.get(gate, gate), url)- failures = 0- if executions_at_retry is None and status not in DEPOT_LIVE_STATES:- sending = True- depot("retry", shown["run"]["run_id"], "--workflow", workflow, "--org", org, "--failed")- executions_at_retry = executions- sys.stdout.write(f"Depot retries the failed jobs: {url}\n")- except (subprocess.SubprocessError, OSError, ValueError, LookupError, TypeError, StopIteration) as error:- failures += 1- sys.stdout.write(f"::warning::Depot CLI call failed: {type(error).__name__}\n")- # A retry that failed is not sent again, because Depot can have accepted it.- if sending or failures >= MAX_REFUSALS:- return None- if clock() - start >= GATE_DEADLINE_MINUTES * 60:- return Progress(Phase.RUNNING, details_url=url)- sleep(30)---def gate_verdict(- reader: CheckReader,- event: Event,- *,- rerun: bool,- retry: Callable[[str, str], Progress | None] = retry_failed_jobs,-) -> Progress:- """The gate verdict that this attempt of the relay job reports.-- A GitHub re-run starts nothing on Depot. So when a re-run finds a gate that already failed- or was cancelled, the relay retries the failed and cancelled Depot jobs and reports the new- verdict. A cancel is not a test verdict, so the re-run asks for the tests again. A failed- prerequisite is not retried, because it fails the same way until a commit fixes it.- """- if rerun:- settled = poll(reader, event, GATE_CHECK, deadline_minutes=0, absent_minutes=0)- target = DEPOT_RUN_URL.match(settled.details_url)- failed = settled.phase == Phase.FINISHED and settled.state != "success"- if target and (failed or settled.phase == Phase.CANCELLED) and not settled.root_failure:- retried = retry(*target.groups())- if retried:- return retried- sys.stdout.write("::warning::Cannot retry on Depot CI. This is the earlier verdict.\n")- return settled- return poll(reader, event, GATE_CHECK, deadline_minutes=GATE_DEADLINE_MINUTES, absent_minutes=15)---def retry_instructions(event: Event, details_url: str, run_id: str = "") -> list[str]:- """The ways to run the tests again. A GitHub re-run retries a failed Depot run, so it is listed with a `run_id`."""- rerun = [- "Re-run this job on GitHub to retry the failed Depot jobs:",- f" gh run rerun {run_id} --repo {event.repo} --failed",- "",- ]- lines = [- f"Backend tests for {event.sha} ran on Depot CI, not GitHub Actions.",- f"Depot run: {details_url or 'not found'}",- "",- *(rerun if run_id else []),- ]- # Labels do not route a merge queue batch, and a push to its branch ends the queue attempt.- if event.merge_queue:- return [- *lines,- "Run on GitHub Actions instead: CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT=0 keeps new merge queue batches there.",- f" gh variable set CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT --repo {event.repo} --body 0",- "Then queue the pull request again.",- ]- return [- *lines,- "Run on GitHub Actions instead: the ci-backend-github label routes the next commit of this PR there.",- f" gh pr edit {event.pr_number} --repo {event.repo} --add-label ci-backend-github",- " git commit --allow-empty -m 'chore: retry backend ci on github actions' && git push",- ]---def relay_gate(result: Progress, event: Event, run_id: str) -> tuple[int, list[str]]:- """The exit code and log lines of the `Django Tests Pass` relay for the gate's progress."""- if result.phase == Phase.FINISHED and result.state == "success":- return 0, []- if result.root_failure:- return 1, [- f"::error::{result.root_failure} failed on Depot (check {result.root_check_id}). "- "Push a fix; a retry will not help."- ]- if result.phase == Phase.FINISHED:- return 1, [- f"::error::Backend tests on Depot CI concluded {result.state}. This step's log lists the retry options.",- *retry_instructions(event, result.details_url, run_id),- ]- if result.phase == Phase.CANCELLED:- return 1, [- f"::error::Depot CI cancelled its run for this event of {event.sha} and started no replacement.",- *retry_instructions(event, result.details_url, run_id),- ]- if result.phase == Phase.DECLINED:- return 1, [f"::error::Depot declined the hand-off for {event.sha} (wait job: {result.state})"]- if result.phase == Phase.ABSENT:- return 1, [f"::error::Depot CI started no run for this event of {event.sha}: no wait job for it"]- return 1, [f"::error::No Depot verdict for {event.sha} within the relay's deadline"]---def main(argv: Sequence[str]) -> int:- if argv[1:] != ["gate"]:- sys.stderr.write("usage: ci_backend_relay.py gate\n")- return 2- env = os.environ- event = Event(- repo=env["REPO"],- sha=env["SHA"],- pr_number=int(env["PR_NUMBER"]),- event_at=env["EVENT_AT"],- merge_queue=is_merge_queue(env.get("HEAD_REF", "")),- )- reader = CheckRunReader(event.repo, event.sha, env["GH_TOKEN"], pr_number=event.pr_number)- try:- result = gate_verdict(reader, event, rerun=env.get("GITHUB_RUN_ATTEMPT", "1") != "1")- except ReadRefusedError as error:- sys.stdout.write(f"::error::{error}\n")- return 1- code, lines = relay_gate(result, event, env.get("GITHUB_RUN_ID", ""))- if code and result.phase == Phase.FINISHED and (match := DEPOT_RUN_URL.match(result.details_url)):- lines += explain(*match.groups())- sys.stdout.writelines(f"{line}\n" for line in lines)- return code---if __name__ == "__main__":- sys.exit(main(sys.argv))diff --git a/.github/scripts/ci_backend_route.py b/.github/scripts/ci_backend_route.pydeleted file mode 100644index 8e89908bd055..000000000000--- a/.github/scripts/ci_backend_route.py+++ /dev/null@@ -1,271 +0,0 @@-#!/usr/bin/env python3-"""Decide which CI engine runs a Backend CI event: GitHub Actions or Depot CI.--Only GitHub Actions runs this script. Its answer drives the `Hand off backend tests-to Depot CI` job, which Depot CI waits for before it runs anything, so Depot never-routes on its own and the tests never run on both engines. Once that job has concluded-for a commit, every later run of the same commit repeats its answer, whatever the-percent or the labels say by then, even after the rollout variable is deleted. A read of-that record that keeps failing stops routing, so a commit cannot run on both engines.--Any other event goes to Depot only after Depot started a run for it. Depot compiles its run-from the pull request's merge ref as soon as the event arrives, while GitHub Actions waits-until GitHub has updated that ref. When the ref is missing, Depot ends the run with no-workflows; when it stays stale, Depot fails to compile. Depot also cancels some runs under-its concurrency policy before they start. Each case leaves the event with no Depot run, so-it stays here.--Merge queue batches have their own percent, CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT, so the-merge gate moves to Depot separately from source pull requests. Batches ignore the routing-labels, so setting that percent to 0 keeps every new batch on GitHub Actions.-"""--import os-import sys-import json-import time-import hashlib-import urllib.error-import urllib.parse-import urllib.request-from collections.abc import Callable-from dataclasses import dataclass-from datetime import UTC, datetime-from typing import Any--from ci_backend_relay import (- EVENT_TIME,- MIRROR_APP_ID,- CheckReader,- CheckRunReader,- ReadFailedError,- ReadRefusedError,- event_check_name,- is_merge_queue,-)--LABEL_FORCE_GITHUB = "ci-backend-github"-LABEL_FORCE_DEPOT = "ci-backend-depot"-PERCENT_VARIABLE = "CI_BACKEND_DEPOT_PERCENT"-HANDOFF_CHECK = "Hand off backend tests to Depot CI"-GITHUB_ACTIONS_APP_ID = 15368-ENGINE_BY_HANDOFF_CONCLUSION = {"success": "depot", "skipped": "github"}-API_ROOT = "https://api.github.com"-API_ATTEMPTS = 3-API_BACKOFF_SECONDS = 5-# The first step of Depot's wait job in .depot/workflows/ci-backend.yml posts this check; change both.-STARTED_JOB = "Depot run started"-# How long after the event Depot's wait job may take to start before the event stays on GitHub Actions.-DEPOT_START_SECONDS = 180-DEPOT_POLL_SECONDS = 10---class HandoffReadError(RuntimeError):- """The durable routing record could not be read, so the event must not be routed."""---@dataclass(frozen=True)-class Decision:- engine: str- reason: str---def parse_percent(raw: str | None) -> int:- value = raw.strip() if raw is not None else ""- # isdecimal() accepts digits int() rejects, and int() refuses very long strings.- if not value.isascii() or not value.isdecimal():- return 0- try:- return min(int(value), 100)- except ValueError:- return 0---def bucket_of(pr_number: int) -> int:- """The pull request's fixed rollout bucket, 0 to 99.-- Pull request numbers are sequential, so `pr_number % 100` would give Depot runs of- consecutive pull requests and then none for the next 95. A hash spreads them while- every push to one pull request keeps its bucket. Python's `hash()` is salted per- process, so it would move a pull request between engines from one run to the next.- """- return int.from_bytes(hashlib.sha256(str(pr_number).encode()).digest()[:8], "big") % 100---def handoff_conclusion(check_runs: list[dict], pr_number: int) -> str | None:- """The hand-off conclusion that already committed an engine to this commit for this pull request.-- A `success` means Depot ran the tests and a `skipped` means GitHub Actions did, and- neither is undone by a later attempt that was cancelled or re-routed, so the first- of those found wins over anything newer. GitHub queues a fresh check for every- attempt, so only concluded ones count, and the API lists every pull request's checks- for a head SHA, so only this one's count.- """- conclusions = [- run.get("conclusion")- for run in sorted(check_runs, key=lambda run: run["id"], reverse=True)- if run.get("status") == "completed"- and any(pr.get("number") == pr_number for pr in run.get("pull_requests") or [])- ]- for committed in ENGINE_BY_HANDOFF_CONCLUSION:- if committed in conclusions:- return committed- return conclusions[0] if conclusions else None---def by_bucket(pr_number: int | None, percent: int, label: str) -> Decision:- if pr_number is None:- return Decision("github", "no pull request number to hash")- bucket = bucket_of(pr_number)- if bucket < percent:- return Decision("depot", f"{label} {bucket} < {percent}%")- return Decision("github", f"{label} {bucket} >= {percent}%")---def decide(- event: str,- percent: int,- pr_number: int | None,- labels: list[str],- is_fork: bool,- is_draft: bool,- prior_handoff: str | None = None,- head_ref: str = "",- merge_queue_percent: int = 0,-) -> Decision:- if event != "pull_request":- return Decision("github", f"{event} events stay on GitHub Actions")- prior_engine = ENGINE_BY_HANDOFF_CONCLUSION.get(prior_handoff or "")- if prior_engine:- return Decision(prior_engine, f"an earlier run of this commit chose {prior_engine}")- if is_merge_queue(head_ref):- return by_bucket(pr_number, merge_queue_percent, "merge queue bucket")- if LABEL_FORCE_GITHUB in labels:- return Decision("github", f"label {LABEL_FORCE_GITHUB}")- if is_fork:- return Decision("github", "fork pull requests never run on Depot")- if is_draft and "no-ci" in labels:- return Decision("github", "no-ci drafts skip on GitHub Actions and run nowhere else")- if LABEL_FORCE_DEPOT in labels:- return Decision("depot", f"label {LABEL_FORCE_DEPOT}")- return by_bucket(pr_number, percent, "bucket")---def fetch_handoff_checks(repo: str, sha: str, token: str, *, opener: Any = None) -> list[dict]:- """GET this commit's hand-off check runs, retrying transient failures only.-- 5xx and network errors are worth another attempt; 4xx is not, because with an- empty rate-limit bucket or a missing permission a retry only spends more of it.- """- query = urllib.parse.urlencode(- {"check_name": HANDOFF_CHECK, "app_id": GITHUB_ACTIONS_APP_ID, "filter": "all", "per_page": 100}- )- url = f"{API_ROOT}/repos/{repo}/commits/{sha}/check-runs?{query}"- request = urllib.request.Request(- url,- headers={- "Accept": "application/vnd.github+json",- "X-GitHub-Api-Version": "2022-11-28",- "Authorization": f"Bearer {token}",- },- )- do_open = opener or urllib.request.urlopen- for attempt in range(1, API_ATTEMPTS + 1):- try:- with do_open(request, timeout=30) as response:- return json.loads(response.read().decode("utf-8"))["check_runs"]- except urllib.error.HTTPError as error:- if error.code < 500 or attempt == API_ATTEMPTS:- raise HandoffReadError(f"GET {url} failed with {error.code}") from error- except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError, TypeError) as error:- if attempt == API_ATTEMPTS:- raise HandoffReadError(f"GET {url} failed: {error}") from error- sys.stdout.write(f"::warning::Cannot read the earlier hand-off yet ({attempt}/{API_ATTEMPTS})\n")- time.sleep(API_BACKOFF_SECONDS * attempt)- raise HandoffReadError(f"GET {url} exhausted {API_ATTEMPTS} attempts")---def depot_started(- reader: CheckReader,- pr_number: int,- event_at: str,- *,- clock: Callable[[], float] = time.time,- sleep: Callable[[float], None] = time.sleep,-) -> bool:- """Whether Depot started a run for this event, waiting up to DEPOT_START_SECONDS after it.-- The started check names the event, so it identifies this event's run and no other.- """- try:- deadline = datetime.strptime(event_at, EVENT_TIME).replace(tzinfo=UTC).timestamp() + DEPOT_START_SECONDS- except ValueError:- return False- name = event_check_name(STARTED_JOB, pr_number, event_at)- while True:- try:- if reader.read(name):- return True- except ReadFailedError:- pass- except ReadRefusedError:- return False- if clock() >= deadline:- return False- sleep(DEPOT_POLL_SECONDS)---def main() -> int:- env = os.environ- event = env.get("EVENT", "")- pr_raw = env.get("PR_NUMBER", "")- pr_number = int(pr_raw) if pr_raw.isdigit() else None- is_fork = env.get("IS_FORK", "false") == "true"- labels = json.loads(env.get("LABELS") or "null") or []-- def route_with(prior_handoff: str | None) -> Decision:- return decide(- event=event,- percent=parse_percent(env.get("PERCENT")),- pr_number=pr_number,- labels=labels,- is_fork=is_fork,- is_draft=env.get("IS_DRAFT", "false") == "true",- prior_handoff=prior_handoff,- head_ref=env.get("HEAD_REF", ""),- merge_queue_percent=parse_percent(env.get("MERGE_QUEUE_PERCENT")),- )-- prior_handoff = None- if event == "pull_request" and pr_number is not None and not is_fork:- try:- prior_handoff = handoff_conclusion(- fetch_handoff_checks(env["REPO"], env["SHA"], env["GH_TOKEN"]), pr_number- )- except HandoffReadError as error:- sys.stdout.write(f"::error::Cannot read the earlier hand-off, so this event is not routed: {error}\n")- return 1- else:- sys.stdout.write(f"::notice::Earlier hand-off on this commit: {prior_handoff or 'none'}\n")- decision = route_with(prior_handoff)- if (- decision.engine == "depot"- and prior_handoff not in ENGINE_BY_HANDOFF_CONCLUSION- and pr_number is not None- and not depot_started(- CheckRunReader(env["REPO"], env["SHA"], env["GH_TOKEN"], pr_number=pr_number, app_ids=(MIRROR_APP_ID,)),- pr_number,- env.get("EVENT_AT", ""),- )- ):- decision = Decision("github", "Depot CI started no run for this event")- sys.stdout.write(f"::notice::Backend CI engine: {decision.engine} ({decision.reason})\n")- output_path = env.get("GITHUB_OUTPUT")- if output_path:- with open(output_path, "a", encoding="utf-8") as handle:- handle.write(f"engine={decision.engine}\nreason={decision.reason}\n")- return 0---if __name__ == "__main__":- sys.exit(main())diff --git a/.github/scripts/ci_flake_overseer.py b/.github/scripts/ci_flake_overseer.pydeleted file mode 100644index 53a3b045d2d6..000000000000--- a/.github/scripts/ci_flake_overseer.py+++ /dev/null@@ -1,552 +0,0 @@-#!/usr/bin/env python3-# ruff: noqa: T201 - CLI output is the contract for this script.--from __future__ import annotations--import os-import re-import sys-import json-import argparse-import subprocess-import urllib.error-import urllib.request-from dataclasses import dataclass-from datetime import datetime-from pathlib import Path-from typing import Literal, cast--JsonObject = dict[str, object]-# Observe-only: we classify failed CI jobs and record what reruns do, but never rerun anything.-DecisionAction = Literal["observe", "skip deterministic", "skip non-test"]--DEFAULT_ALLOWED_WORKFLOWS = ("Backend CI", "Dagster CI", "E2E CI Playwright")-DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"-DECISION_EVENT = "ci_flake_overseer_decision"-OUTCOME_EVENT = "ci_flake_overseer_rerun_outcome"---def compile_patterns(*patterns: str) -> tuple[re.Pattern[str], ...]:- return tuple(re.compile(pattern, re.IGNORECASE) for pattern in patterns)---def compile_classified(*rules: tuple[str, str]) -> tuple[tuple[re.Pattern[str], str], ...]:- return tuple((re.compile(pattern, re.IGNORECASE), failure_class) for pattern, failure_class in rules)---# Jobs/steps whose failures a rerun can't fix — excluded from the flake measurement.-# Each rule carries a stable failure-class key (matching `hogli ci:preflight` check-# keys where the two overlap) so dashboards can group deterministic failures that-# reached CI and join them against local preflight interceptions.-DETERMINISTIC_JOB_RULES = compile_classified(- (r"\brepo checks?\b", "repo-checks"),- (r"\bopenapi\b", "openapi"),- (r"\bmigrations?\b", "migrations"),- (r"\blint\b", "lint"),- (r"\btype ?check\b", "typecheck"),- (r"\btypescript\b", "typecheck"),- (r"\bvisual\b", "visual-review"),- (r"\bsnapshot\b", "snapshot"),- (r"\bstorybook\b", "storybook"),- (r"\btach\b", "module-boundaries"),- (r"\bimport-linter\b", "module-boundaries"),-)--DETERMINISTIC_STEP_RULES = compile_classified(- (r"\bcheck module boundaries\b", "module-boundaries"),- (r"\bproduct facade enforcement\b", "module-boundaries"),- (r"\bopenapi\b", "openapi"),- (r"\bmigrations?\b", "migrations"),- (r"\blint\b", "lint"),- (r"\btype ?check\b", "typecheck"),- (r"\btypescript\b", "typecheck"),- (r"\bvisual review\b", "visual-review"),- (r"\bsnapshot\b", "snapshot"),- (r"\bverify changed playwright tests are stable\b", "playwright-stability"),- (r"\bverify new snapshots\b", "snapshot"),-)--TEST_STEP_PATTERNS = compile_patterns(- r"\brun core tests\b",- r"\brun temporal tests\b",- r"\brun product tests\b",- r"\brun dagster tests\b",- r"\brun playwright tests\b",-)--TEST_JOB_PATTERNS = compile_patterns(- r"\bdjango tests\b",- r"\bproduct tests\b",- r"\bdagster tests\b",- r"\bplaywright e2e tests\b",-)---class ExternalCommandError(Exception):- pass---@dataclass(frozen=True)-class Step:- name: str- conclusion: str | None---@dataclass(frozen=True)-class Job:- id: int- name: str- conclusion: str | None- run_attempt: int- html_url: str- # A genuine re-run advances started_at; a carried-over job keeps the prior attempt's value byte-for-byte- # (run_attempt reports the latest attempt for every job either way), so started_at is the re-run signal.- started_at: str | None = None- steps: tuple[Step, ...] = ()-- def failed_step_names(self) -> tuple[str, ...]:- return tuple(step.name for step in self.steps if step.conclusion in {"failure", "timed_out"})---@dataclass(frozen=True)-class WorkflowRun:- id: int- workflow_id: int- name: str- conclusion: str | None- head_sha: str- run_attempt: int- html_url: str- head_branch: str = ""- event: str = ""- # Empty `pull_requests` (e.g. fork PRs) collapses to None; head_branch stays the primary join key.- pr_number: int | None = None---@dataclass(frozen=True)-class Decision:- action: DecisionAction- reason: str- job: Job- # Which signal classified an `observe`: the job name or a failed step. Surfaces silent allowlist drift.- classified_via: str | None = None- # Canonical failure-class key for `skip deterministic` decisions (see the rule tables above).- failure_class: str | None = None---def as_str(value: object) -> str | None:- return value if isinstance(value, str) else None---def as_int(value: object) -> int | None:- if isinstance(value, int):- return value- if isinstance(value, str) and value.isdigit():- return int(value)- return None---def as_object(value: object) -> JsonObject:- return cast(JsonObject, value) if isinstance(value, dict) else {}---def as_object_list(value: object) -> list[JsonObject]:- if not isinstance(value, list):- return []- return [cast(JsonObject, item) for item in value if isinstance(item, dict)]---def as_bool_string(value: object) -> bool:- return str(value).strip().lower() in {"1", "true", "yes", "on"}---def run_command(args: tuple[str, ...], timeout_seconds: int = 30) -> str:- try:- result = subprocess.run(args, capture_output=True, text=True, timeout=timeout_seconds)- except (FileNotFoundError, subprocess.TimeoutExpired) as exc:- raise ExternalCommandError(str(exc)) from exc- if result.returncode != 0:- message = (result.stderr or result.stdout).strip() or "command failed"- raise ExternalCommandError(message)- return result.stdout---def gh_json(repo: str, path: str) -> JsonObject:- raw = run_command(("gh", "api", f"repos/{repo}/{path}"), timeout_seconds=45)- parsed = json.loads(raw)- return parsed if isinstance(parsed, dict) else {}---def pr_number_from_object(raw: JsonObject) -> int | None:- pulls = as_object_list(raw.get("pull_requests"))- return as_int(pulls[0].get("number")) if pulls else None---def workflow_run_from_object(raw: JsonObject) -> WorkflowRun:- return WorkflowRun(- id=as_int(raw.get("id")) or 0,- workflow_id=as_int(raw.get("workflow_id")) or 0,- name=as_str(raw.get("name")) or as_str(raw.get("workflow_name")) or "",- conclusion=as_str(raw.get("conclusion")),- head_sha=as_str(raw.get("head_sha")) or "",- run_attempt=as_int(raw.get("run_attempt")) or 1,- html_url=as_str(raw.get("html_url")) or "",- head_branch=as_str(raw.get("head_branch")) or "",- event=as_str(raw.get("event")) or "",- pr_number=pr_number_from_object(raw),- )---def workflow_run_from_event(path: Path) -> WorkflowRun | None:- try:- parsed = json.loads(path.read_text())- except (FileNotFoundError, json.JSONDecodeError):- return None- if not isinstance(parsed, dict):- return None- workflow_run = as_object(parsed.get("workflow_run"))- return workflow_run_from_object(workflow_run) if workflow_run else None---def job_from_object(raw: JsonObject, run_attempt: int) -> Job:- return Job(- id=as_int(raw.get("id")) or 0,- name=as_str(raw.get("name")) or "",- conclusion=as_str(raw.get("conclusion")),- # Trust the job's own attempt over the requested one — the fallback endpoint returns the latest attempt.- run_attempt=as_int(raw.get("run_attempt")) or run_attempt,- html_url=as_str(raw.get("html_url")) or "",- started_at=as_str(raw.get("started_at")),- steps=tuple(- Step(name=as_str(step.get("name")) or "", conclusion=as_str(step.get("conclusion")))- for step in as_object_list(raw.get("steps"))- ),- )---def fetch_jobs(repo: str, run_id: int, run_attempt: int, *, strict: bool = False) -> tuple[Job, ...]:- jobs: list[Job] = []- page = 1- while True:- try:- raw = gh_json(repo, f"actions/runs/{run_id}/attempts/{run_attempt}/jobs?per_page=100&page={page}")- except ExternalCommandError:- # The non-attempt endpoint returns the LATEST attempt, so it can't stand in for an arbitrary- # prior attempt. Strict callers need exact rows, so fail closed (drop any partial pages) rather- # than return a truncated set or a masquerading attempt.- if strict:- return ()- raw = gh_json(repo, f"actions/runs/{run_id}/jobs?per_page=100&page={page}")- page_jobs = as_object_list(raw.get("jobs"))- jobs.extend(job_from_object(job, run_attempt) for job in page_jobs)- if len(page_jobs) < 100:- return tuple(jobs)- page += 1---def deterministic_match(job: Job) -> tuple[str, str] | None:- for pattern, failure_class in DETERMINISTIC_JOB_RULES:- if pattern.search(job.name):- return f"job name matches deterministic rule `{pattern.pattern}`", failure_class- for step_name in job.failed_step_names():- for pattern, failure_class in DETERMINISTIC_STEP_RULES:- if pattern.search(step_name):- return f"failed step `{step_name}` matches deterministic rule `{pattern.pattern}`", failure_class- return None---def test_failure_source(job: Job) -> str | None:- if any(pattern.search(job.name) for pattern in TEST_JOB_PATTERNS):- return "job_name"- if any(pattern.search(step_name) for step_name in job.failed_step_names() for pattern in TEST_STEP_PATTERNS):- return "step"- return None---def is_test_job_failure(job: Job) -> bool:- return test_failure_source(job) is not None---def classify_job(job: Job) -> Decision:- deterministic = deterministic_match(job)- if deterministic is not None:- reason, failure_class = deterministic- return Decision(action="skip deterministic", reason=reason, job=job, failure_class=failure_class)- source = test_failure_source(job)- if source is None:- return Decision(action="skip non-test", reason="failed job or step is not an allowlisted test runner", job=job)- return Decision(- action="observe", reason="test job failure tracked for rerun outcome", job=job, classified_via=source- )---def classify_failed_jobs(repo: str, run_id: int, run_attempt: int) -> tuple[Decision, ...]:- return tuple(- classify_job(job) for job in fetch_jobs(repo, run_id, run_attempt) if job.conclusion in {"failure", "timed_out"}- )---def base_event_properties(repo: str, workflow_run: WorkflowRun) -> JsonObject:- return {- "repo": repo,- "workflow_name": workflow_run.name,- "run_id": workflow_run.id,- "run_url": workflow_run.html_url,- "head_sha": workflow_run.head_sha,- "head_branch": workflow_run.head_branch,- "pr_number": workflow_run.pr_number,- # Master failures block the deploy gate; PR failures only annoy the author — keep them separable.- "is_master": workflow_run.head_branch in {"master", "main"},- "trigger_event": workflow_run.event,- # Reuse the project's existing workflow_run group so events roll up per CI run.- "$groups": {"workflow_run": str(workflow_run.id)},- }---def build_decision_events(repo: str, workflow_run: WorkflowRun, decisions: tuple[Decision, ...]) -> list[JsonObject]:- events: list[JsonObject] = []- for decision in decisions:- job = decision.job- properties = base_event_properties(repo, workflow_run)- properties.update(- {- "action": decision.action,- "reason": decision.reason,- "classified_via": decision.classified_via,- "failure_class": decision.failure_class,- "job_name": job.name,- "job_id": job.id,- "job_url": job.html_url,- "job_conclusion": job.conclusion,- "failed_steps": list(job.failed_step_names()),- "run_attempt": job.run_attempt,- # Dedupe duplicate workflow_run deliveries so the count panels stay accurate.- "$insert_id": f"decision:{workflow_run.id}:{workflow_run.run_attempt}:{job.id}",- }- )- events.append({"event": DECISION_EVENT, "distinct_id": repo, "properties": properties})- return events---def rerun_outcome_label(conclusion: str | None) -> str:- if conclusion == "success":- return "cleared"- if conclusion in {"failure", "timed_out"}:- return "still_failing"- return "unknown"---def parse_timestamp(value: str | None) -> datetime | None:- if not value:- return None- try:- # Normalize a trailing `Z` for parsers that don't accept the military-zulu suffix.- return datetime.fromisoformat(value.replace("Z", "+00:00"))- except ValueError:- return None---def job_was_reexecuted(prior_job: Job, current_job: Job) -> bool:- # A real re-run advances started_at; a carried-over job keeps the prior attempt's value. Compare parsed- # instants, not raw strings, so a `+00:00` vs `Z` form can't misorder. Missing/unparseable timestamps- # can't prove re-execution, so treat them as not re-run rather than guess.- prior = parse_timestamp(prior_job.started_at)- current = parse_timestamp(current_job.started_at)- if prior is None or current is None:- return False- return current > prior---def rerun_outcome(prior_job: Job, current_job: Job | None) -> str:- # `current_job is None` covers a job missing from the current attempt or an ambiguous (duplicated) name.- if current_job is None:- return "unknown"- if not job_was_reexecuted(prior_job, current_job):- return "not_rerun"- return rerun_outcome_label(current_job.conclusion)---def index_unique_jobs_by_name(jobs: tuple[Job, ...]) -> dict[str, Job]:- # Job names are unique within an attempt; if one ever repeats, drop it so an ambiguous match becomes- # `unknown` instead of silently picking the wrong row.- by_name: dict[str, Job] = {}- ambiguous: set[str] = set()- for job in jobs:- if job.name in by_name:- ambiguous.add(job.name)- by_name[job.name] = job- for name in ambiguous:- by_name.pop(name, None)- return by_name---def report_rerun_outcomes(repo: str, workflow_run: WorkflowRun) -> list[JsonObject]:- # Outcomes only exist once a re-run attempt has completed. Find the test-job failures from the prior- # attempt, then read how they concluded this attempt — the base-rate signal: do reruns clear flakes?- if workflow_run.run_attempt <= 1:- return []- prior_attempt = workflow_run.run_attempt - 1- # Fetch the prior attempt strict: the non-attempt fallback returns the LATEST attempt, which would alias- # as "prior" and collapse the started_at re-execution check. The current attempt IS the latest, so it- # keeps the fallback (the non-attempt endpoint returns it correctly). Both sides dedupe ambiguous names.- prior_observed = {- name: job- for name, job in index_unique_jobs_by_name(- fetch_jobs(repo, workflow_run.id, prior_attempt, strict=True)- ).items()- if job.conclusion in {"failure", "timed_out"} and classify_job(job).action == "observe"- }- if not prior_observed:- return []- current_by_name = index_unique_jobs_by_name(fetch_jobs(repo, workflow_run.id, workflow_run.run_attempt))- events: list[JsonObject] = []- for job_name in sorted(prior_observed):- prior_job = prior_observed[job_name]- current_job = current_by_name.get(job_name)- properties = base_event_properties(repo, workflow_run)- properties.update(- {- "outcome": rerun_outcome(prior_job, current_job),- "current_conclusion": current_job.conclusion if current_job else None,- "prior_conclusion": prior_job.conclusion,- "job_name": job_name,- "failed_steps": list(prior_job.failed_step_names()),- "prior_attempt": prior_attempt,- "attempt": workflow_run.run_attempt,- # Dedupe duplicate workflow_run deliveries so the base-rate counts stay accurate.- "$insert_id": f"outcome:{workflow_run.id}:{workflow_run.run_attempt}:{job_name}",- }- )- events.append({"event": OUTCOME_EVENT, "distinct_id": repo, "properties": properties})- return events---def capture_events(api_key: str, host: str, events: list[JsonObject], timeout_seconds: int = 10) -> None:- if not api_key or not events:- return- payload = json.dumps({"api_key": api_key, "batch": events}).encode()- request = urllib.request.Request(- f"{host.rstrip('/')}/batch/",- data=payload,- headers={"Content-Type": "application/json"},- method="POST",- )- try:- # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected -- host is CI config, not request input- with urllib.request.urlopen(request, timeout=timeout_seconds):- pass- except (urllib.error.URLError, TimeoutError, OSError) as exc:- print(f"::warning title=CI flake overseer telemetry failed::{escape_annotation(str(exc))}")---def escape_annotation(value: str) -> str:- return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")---def escape_table(value: str) -> str:- return value.replace("|", "\\|").replace("\n", " ")---def markdown_link(label: str, url: str) -> str:- safe_label = escape_table(label)- return f"[{safe_label}]({url})" if url else safe_label---def format_annotation(decision: Decision) -> str:- detail = f"{decision.job.name}: {decision.reason}"- return f"::notice title=CI flake overseer: {decision.action}::{escape_annotation(detail)}"---def write_summary(path: Path | None, workflow_run: WorkflowRun, decisions: tuple[Decision, ...]) -> None:- if path is None:- return- lines = [- "### CI flake overseer (observe-only)",- "",- f"- Run: {markdown_link(workflow_run.name or str(workflow_run.id), workflow_run.html_url)}",- f"- Head SHA: `{workflow_run.head_sha}`",- f"- Attempt: `{workflow_run.run_attempt}`",- "",- "| Job | Decision | Reason |",- "| --- | --- | --- |",- ]- for decision in decisions:- lines.append(- f"| {markdown_link(decision.job.name, decision.job.html_url)} | "- f"`{decision.action}` | {escape_table(decision.reason)} |"- )- if not decisions:- lines.append("| No failed jobs | `skip non-test` | workflow has no failed jobs to inspect |")- with path.open("a") as summary:- summary.write("\n".join(lines))- summary.write("\n")---def parse_workflows(value: str | None) -> tuple[str, ...]:- if not value:- return DEFAULT_ALLOWED_WORKFLOWS- workflows = tuple(item.strip() for item in value.split(",") if item.strip())- return workflows or DEFAULT_ALLOWED_WORKFLOWS---def resolve_workflow_run(args: argparse.Namespace) -> WorkflowRun:- if args.event_path:- from_event = workflow_run_from_event(Path(args.event_path))- if from_event is not None and (args.run_id is None or from_event.id == args.run_id):- return from_event- if args.run_id is None:- raise ValueError("--run-id is required when the event payload does not contain workflow_run")- return workflow_run_from_object(gh_json(args.repo, f"actions/runs/{args.run_id}"))---def parse_args(argv: list[str]) -> argparse.Namespace:- parser = argparse.ArgumentParser(description="Observe failed CI test jobs and record what reruns do.")- parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "PostHog/posthog"))- parser.add_argument("--run-id", type=int)- parser.add_argument("--event-path", default=os.environ.get("GITHUB_EVENT_PATH"))- parser.add_argument(- "--enabled", action="store_true", default=as_bool_string(os.environ.get("CI_FLAKE_OVERSEER_ENABLED", ""))- )- parser.add_argument("--allowed-workflows", default=os.environ.get("CI_FLAKE_OVERSEER_WORKFLOWS"))- parser.add_argument("--summary-path", default=os.environ.get("GITHUB_STEP_SUMMARY"))- # Reuses the DevEx project token already wired into other CI workflows; telemetry no-ops when absent.- parser.add_argument("--posthog-api-key", default=os.environ.get("POSTHOG_DEVEX_PROJECT_API_TOKEN", ""))- parser.add_argument("--posthog-host", default=os.environ.get("POSTHOG_DEVEX_PROJECT_HOST") or DEFAULT_POSTHOG_HOST)- parser.add_argument("--posthog-timeout-seconds", type=int, default=10)- return parser.parse_args(argv)---def main(argv: list[str]) -> int:- args = parse_args(argv)- summary_path = Path(args.summary_path) if args.summary_path else None- try:- workflow_run = resolve_workflow_run(args)- except Exception as exc:- print(f"::warning title=CI flake overseer failed closed::{escape_annotation(str(exc))}")- return 0-- if workflow_run.name not in parse_workflows(args.allowed_workflows):- print(f"::notice title=CI flake overseer skipped::Workflow `{workflow_run.name}` is not allowlisted")- return 0- if not args.enabled:- print("::notice title=CI flake overseer disabled::Set CI_FLAKE_OVERSEER_ENABLED=true to record observations")- return 0-- # On a re-run attempt, record whether the prior attempt's test failures cleared on rerun.- events = report_rerun_outcomes(args.repo, workflow_run)-- if workflow_run.conclusion in {"failure", "timed_out"}:- decisions = classify_failed_jobs(args.repo, workflow_run.id, workflow_run.run_attempt)- for decision in decisions:- print(format_annotation(decision))- write_summary(summary_path, workflow_run, decisions)- events.extend(build_decision_events(args.repo, workflow_run, decisions))- else:- print(f"::notice title=CI flake overseer skipped::Workflow conclusion is `{workflow_run.conclusion}`")-- capture_events(args.posthog_api_key, args.posthog_host, events, args.posthog_timeout_seconds)- return 0---if __name__ == "__main__":- raise SystemExit(main(sys.argv[1:]))diff --git a/.github/scripts/compare-ci-runners.py b/.github/scripts/compare-ci-runners.pydeleted file mode 100755index 50c89d90c173..000000000000--- a/.github/scripts/compare-ci-runners.py+++ /dev/null@@ -1,526 +0,0 @@-#!/usr/bin/env python3-# ruff: noqa: T201 print is the intended output of this CLI-#-# THIS SCRIPT IS NO LONGER USED!-# The Blacksmith trial period is over and we no longer run Blacksmith-# runners alongside Depot, so there is nothing to pair.-#-# Pair Depot and Blacksmith job durations for the 7-day CI trial.-#-# Usage:-# python3 .github/scripts/compare-ci-runners.py --days 7 > report.md-# python3 .github/scripts/compare-ci-runners.py --since 2026-04-20T18:00:00Z --until 2026-04-21T10:30:00Z > report.md-# python3 .github/scripts/compare-ci-runners.py --days 7 --format csv > pairs.csv-#-# Requires: `gh` CLI authenticated, `jq` not required (we parse JSON in Python).-# Pulls jobs for listed workflows, groups by (workflow, base_job_name, sha),-# splits each group into depot-runner vs blacksmith-runner, reports paired deltas.-#-# How pairing works:-# - Matrix-expanded jobs have names like "Python code quality (depot-ubuntu-latest)"-# and "Python code quality (blacksmith-2vcpu-ubuntu-2404)". Base name stripped-# by removing trailing " (<runner>)".-# - Dedicated shadow jobs use the suffix "-blacksmith" or "blacksmith shadow" in name.-# - Pairs: same (workflow, base_name, sha) → one depot duration + one blacksmith duration.-#-# Conclusions:-# - Speedup uses only success↔success pairs (mixed conclusions would be apples-to-oranges).-# - Failure rates are tracked per runner across ALL observed runs (paired or not).-# - Singletons (runs that only appeared on one runner for a given SHA) are reported separately.--import re-import sys-import json-import time-import argparse-import statistics-import subprocess-from collections import defaultdict-from concurrent.futures import ThreadPoolExecutor, as_completed-from datetime import UTC, datetime, timedelta-from pathlib import Path--WORKFLOWS = [- "ci-backend.yml",- "ci-dagster.yml",- "ci-e2e-playwright.yml",- "ci-mcp.yml",- "ci-nodejs.yml",- "ci-proto.yml",- "ci-python.yml",- "ci-rust-flags-integration.yml",- "ci-rust.yml",- "ci-storybook.yml",-]--# Matches a runner label at the end of a job name, either alone in trailing parens-# ("Name (depot-foo)") or as the last comma-separated item ("Name (1/3, depot-foo)").-# The optional prefix group lets us keep shard/group context so matrix jobs that mix-# shard and runner in the same parens still pair across providers.-RUNNER_LABEL_RE = re.compile(- r"\s*\((?:(?P<prefix>[^()]*?),\s*)?(?P<runner>depot-[a-z0-9.-]+|blacksmith-[a-z0-9.-]+)\s*\)\s*$"-)---DEFAULT_CACHE_DIR = Path(".cache/compare-ci-runners")-SKIPPED_RUN_CONCLUSIONS = {"skipped", "cancelled", "action_required"}---def is_retryable_gh_error(stderr: str) -> bool:- return "HTTP 5" in stderr or "Server Error" in stderr or "connection reset" in stderr---def run_gh(args: list[str], attempts: int = 3) -> str:- for attempt in range(1, attempts + 1):- try:- r = subprocess.run(["gh", *args], capture_output=True, text=True, check=True)- return r.stdout- except subprocess.CalledProcessError as e:- stderr = e.stderr or ""- if attempt < attempts and is_retryable_gh_error(stderr):- time.sleep(2 * attempt)- continue- if stderr:- print(stderr.strip(), file=sys.stderr)- raise- raise RuntimeError("unreachable")---def parse_utc_datetime(value: str) -> datetime:- try:- dt = datetime.fromisoformat(value.replace("Z", "+00:00"))- except ValueError as e:- raise argparse.ArgumentTypeError(f"invalid datetime: {value}") from e- if dt.tzinfo is None:- dt = dt.replace(tzinfo=UTC)- return dt.astimezone(UTC)---def created_dates(since: datetime, until: datetime) -> list[str]:- day = since.date()- last_day = until.date()- dates = []- while day <= last_day:- dates.append(day.isoformat())- day += timedelta(days=1)- return dates---def safe_cache_key(value: str) -> str:- return re.sub(r"[^A-Za-z0-9_.-]+", "_", value)---def load_cached_run_list(cache_dir: Path, workflow: str, created: str) -> list[dict] | None:- cache_path = cache_dir / "runs" / f"{safe_cache_key(workflow)}-{created}.json"- try:- return json.loads(cache_path.read_text())- except FileNotFoundError:- return None- except json.JSONDecodeError:- return None---def save_cached_run_list(cache_dir: Path, workflow: str, created: str, runs: list[dict]) -> None:- cache_path = cache_dir / "runs" / f"{safe_cache_key(workflow)}-{created}.json"- cache_path.parent.mkdir(parents=True, exist_ok=True)- cache_path.write_text(json.dumps(runs, sort_keys=True))---def list_runs_for_date(workflow: str, created: str, cache_dir: Path, use_cache: bool) -> tuple[list[dict], bool]:- if use_cache:- cached_runs = load_cached_run_list(cache_dir, workflow, created)- if cached_runs is not None:- return cached_runs, True- out = run_gh(- [- "run",- "list",- "--workflow",- workflow,- "--created",- created,- "-L",- "1000",- "--json",- "databaseId,headSha,createdAt,conclusion,event",- ]- )- runs = json.loads(out)- if use_cache:- save_cached_run_list(cache_dir, workflow, created, runs)- return runs, False---def list_recent_runs(- workflow: str,- since: datetime,- until: datetime,- include_cancelled: bool,- cache_dir: Path,- use_cache: bool,-) -> tuple[list[dict], dict[str, int], dict[str, int]]:- # Use GitHub's created filter to avoid walking unrelated pages, then apply- # exact timestamp bounds locally because --created is day-granular here.- runs_by_id: dict[int, dict] = {}- skipped_by_conclusion: dict[str, int] = defaultdict(int)- cache_stats = {"run_list_cache_hits": 0, "run_list_api_fetches": 0}- allowed_skipped_conclusions = SKIPPED_RUN_CONCLUSIONS - ({"cancelled"} if include_cancelled else set())- for created in created_dates(since, until):- runs, from_cache = list_runs_for_date(workflow, created, cache_dir, use_cache)- cache_stats["run_list_cache_hits" if from_cache else "run_list_api_fetches"] += 1- for r in runs:- run_id = r.get("databaseId")- if not run_id or run_id in runs_by_id:- continue- try:- ts = datetime.fromisoformat(r["createdAt"].replace("Z", "+00:00"))- except Exception:- continue- if ts < since or ts > until:- continue- conclusion = r.get("conclusion") or "unknown"- if conclusion in allowed_skipped_conclusions:- skipped_by_conclusion[conclusion] += 1- continue- runs_by_id[run_id] = r- return list(runs_by_id.values()), dict(skipped_by_conclusion), cache_stats---def load_cached_jobs(cache_dir: Path, run_id: int) -> list[dict] | None:- cache_path = cache_dir / "jobs" / f"{run_id}.json"- try:- return json.loads(cache_path.read_text())- except FileNotFoundError:- return None- except json.JSONDecodeError:- return None---def save_cached_jobs(cache_dir: Path, run_id: int, jobs: list[dict]) -> None:- cache_path = cache_dir / "jobs" / f"{run_id}.json"- cache_path.parent.mkdir(parents=True, exist_ok=True)- cache_path.write_text(json.dumps(jobs, sort_keys=True))---def get_jobs(run_id: int, repo: str, cache_dir: Path, use_cache: bool) -> tuple[list[dict], bool]:- if use_cache:- cached_jobs = load_cached_jobs(cache_dir, run_id)- if cached_jobs is not None:- return cached_jobs, True- out = run_gh(- [- "api",- "--paginate",- "--slurp",- f"repos/{repo}/actions/runs/{run_id}/jobs",- ]- )- pages = json.loads(out)- jobs = []- for page in pages:- for j in page.get("jobs", []):- jobs.append(- {- "name": j.get("name"),- "started_at": j.get("started_at"),- "completed_at": j.get("completed_at"),- "conclusion": j.get("conclusion"),- }- )- if use_cache:- save_cached_jobs(cache_dir, run_id, jobs)- return jobs, False---def classify_runner(job_name: str) -> tuple[str, str]:- """- Returns (base_name, runner_provider). runner_provider is 'depot', 'blacksmith',- or 'unknown'. base_name strips the trailing runner label if present, or the- 'blacksmith shadow' / '-blacksmith' marker.- """- # Matrix-expanded form: "Name (depot-foo)", "Name (blacksmith-foo)", or the- # mixed form "Name (1/3, depot-foo)" where runner shares parens with shard/group.- m = RUNNER_LABEL_RE.search(job_name)- if m:- provider = "depot" if m.group("runner").startswith("depot") else "blacksmith"- prefix = m.group("prefix")- head = job_name[: m.start()].rstrip()- # Preserve non-runner parens content (e.g. shard) so depot and blacksmith- # variants collapse to the same base name.- base = f"{head} ({prefix.strip()})" if prefix is not None else head- return base, provider-- # Dedicated shadow form: "... (blacksmith shadow ...)" or "...-blacksmith"- if "blacksmith shadow" in job_name.lower():- base = re.sub(r"\s*\(blacksmith shadow[^)]*\)\s*", "", job_name).strip()- return base, "blacksmith"- if job_name.endswith("-blacksmith"):- return job_name[: -len("-blacksmith")], "blacksmith"-- # Unrecognized job naming should not be attributed to either provider.- return job_name, "unknown"---def duration_seconds(started: str, completed: str) -> float | None:- if not started or not completed:- return None- try:- s = datetime.fromisoformat(started.replace("Z", "+00:00"))- c = datetime.fromisoformat(completed.replace("Z", "+00:00"))- return (c - s).total_seconds()- except Exception:- return None---def get_repo() -> str:- out = run_gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"])- return out.strip()---def collect_jobs_for_runs(- runs: list[dict], repo: str, cache_dir: Path, use_cache: bool, workers: int-) -> tuple[list[tuple[dict, list[dict]]], dict[str, int]]:- stats = {"cache_hits": 0, "api_fetches": 0, "errors": 0}- if not runs:- return [], stats-- results: list[tuple[dict, list[dict]]] = []- max_workers = max(1, workers)- with ThreadPoolExecutor(max_workers=max_workers) as executor:- futures = {executor.submit(get_jobs, run["databaseId"], repo, cache_dir, use_cache): run for run in runs}- for future in as_completed(futures):- run = futures[future]- try:- jobs, from_cache = future.result()- except subprocess.CalledProcessError:- stats["errors"] += 1- continue- stats["cache_hits" if from_cache else "api_fetches"] += 1- results.append((run, jobs))- return results, stats---def derive_stats(by_sha: dict[str, dict[str, dict]]) -> dict:- """Reduce a (wf, base) sha→provider→{sec,conclusion} map into summary stats."""- depot_total = depot_fail = 0- bs_total = bs_fail = 0- success_pairs: list[tuple[float, float]] = []- mixed_pairs = 0- depot_only = bs_only = 0- for providers in by_sha.values():- d = providers.get("depot")- b = providers.get("blacksmith")- if d:- depot_total += 1- if d["conclusion"] == "failure":- depot_fail += 1- if b:- bs_total += 1- if b["conclusion"] == "failure":- bs_fail += 1- if d and b:- if d["conclusion"] == "success" and b["conclusion"] == "success":- success_pairs.append((d["sec"], b["sec"]))- else:- mixed_pairs += 1- elif d:- depot_only += 1- elif b:- bs_only += 1- return {- "depot_total": depot_total,- "depot_fail": depot_fail,- "bs_total": bs_total,- "bs_fail": bs_fail,- "success_pairs": success_pairs,- "mixed_pairs": mixed_pairs,- "depot_only": depot_only,- "bs_only": bs_only,- }---def main() -> None:- ap = argparse.ArgumentParser()- ap.add_argument("--days", type=int, default=7)- ap.add_argument("--since", type=parse_utc_datetime)- ap.add_argument("--until", type=parse_utc_datetime)- ap.add_argument("--format", choices=["md", "csv"], default="md")- ap.add_argument("--repo", help="GitHub repo in OWNER/REPO form. Defaults to gh's current repo.")- ap.add_argument(- "--workflow",- action="append",- dest="workflows",- help="Workflow file to include. Can be passed multiple times. Defaults to all tracked workflows.",- )- ap.add_argument("--workers", type=int, default=8)- ap.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR)- ap.add_argument("--no-cache", action="store_true")- ap.add_argument(- "--include-cancelled",- action="store_true",- help="Fetch jobs for cancelled workflow runs. Useful for forensics, but noisy for runner comparison.",- )- args = ap.parse_args()- if args.until and not args.since:- ap.error("--until requires --since")-- repo = args.repo or get_repo()- until = args.until or datetime.now(UTC)- since = args.since or (until - timedelta(days=args.days))- if since > until:- ap.error("--since must be before --until")- use_cache = not args.no_cache- print(f"# Collecting jobs from {repo} from {since.isoformat()} through {until.isoformat()}", file=sys.stderr)-- # observations[(workflow, base_name)][sha][provider] = {"sec": dur, "conclusion": conc}- observations: dict[tuple[str, str], dict[str, dict[str, dict]]] = defaultdict(lambda: defaultdict(dict))- total_runs = total_cache_hits = total_api_fetches = total_errors = 0- total_run_list_cache_hits = total_run_list_api_fetches = 0- total_skipped_by_conclusion: dict[str, int] = defaultdict(int)-- workflows = args.workflows or WORKFLOWS- for wf in workflows:- print(f"# {wf}", file=sys.stderr)- try:- runs, skipped_by_conclusion, run_list_stats = list_recent_runs(- wf, since, until, args.include_cancelled, args.cache_dir, use_cache- )- except subprocess.CalledProcessError as e:- print(f" SKIP ({e})", file=sys.stderr)- continue- total_run_list_cache_hits += run_list_stats["run_list_cache_hits"]- total_run_list_api_fetches += run_list_stats["run_list_api_fetches"]- for conclusion, count in skipped_by_conclusion.items():- total_skipped_by_conclusion[conclusion] += count- total_runs += len(runs)- run_jobs, fetch_stats = collect_jobs_for_runs(runs, repo, args.cache_dir, use_cache, args.workers)- total_cache_hits += fetch_stats["cache_hits"]- total_api_fetches += fetch_stats["api_fetches"]- total_errors += fetch_stats["errors"]- print(- f" runs={len(runs)} run_list_cache_hits={run_list_stats['run_list_cache_hits']} "- f"run_list_api_fetches={run_list_stats['run_list_api_fetches']} "- f"job_cache_hits={fetch_stats['cache_hits']} job_api_fetches={fetch_stats['api_fetches']} "- f"errors={fetch_stats['errors']}",- file=sys.stderr,- )- for run, jobs in run_jobs:- sha = run["headSha"]- for j in jobs:- base, provider = classify_runner(j["name"])- dur = duration_seconds(j["started_at"], j["completed_at"])- if dur is None:- continue- if j["conclusion"] not in ("success", "failure"):- continue # skipped/cancelled not useful- observations[(wf, base)][sha][provider] = {- "sec": dur,- "conclusion": j["conclusion"],- }- skipped_summary = ", ".join(- f"{conclusion}={count}" for conclusion, count in sorted(total_skipped_by_conclusion.items())- )- print(- "# Summary: "- f"runs_fetched={total_runs} run_list_cache_hits={total_run_list_cache_hits} "- f"run_list_api_fetches={total_run_list_api_fetches} job_cache_hits={total_cache_hits} "- f"job_api_fetches={total_api_fetches} "- f"errors={total_errors} skipped_before_jobs=({skipped_summary or 'none'})",- file=sys.stderr,- )-- if args.format == "csv":- print("workflow,job,sha,depot_sec,blacksmith_sec,depot_conclusion,blacksmith_conclusion,delta_sec,speedup")- for (wf, base), by_sha in sorted(observations.items()):- for sha, providers in by_sha.items():- d = providers.get("depot")- b = providers.get("blacksmith")- d_sec = f"{d['sec']:.1f}" if d else ""- b_sec = f"{b['sec']:.1f}" if b else ""- d_conc = d["conclusion"] if d else ""- b_conc = b["conclusion"] if b else ""- if d and b and d["conclusion"] == "success" and b["conclusion"] == "success" and b["sec"] > 0:- delta = f"{d['sec'] - b['sec']:.1f}"- speedup = f"{d['sec'] / b['sec']:.2f}"- else:- delta = ""- speedup = ""- print(f'"{wf}","{base}",{sha},{d_sec},{b_sec},{d_conc},{b_conc},{delta},{speedup}')- return-- # Markdown summary- print("# Blacksmith vs Depot — paired CI job durations")- print(f"\nCollection window: `{since.isoformat()}` through `{until.isoformat()}`. Repo: `{repo}`.\n")- print(- "Each row is an `(workflow, job)` with at least one same-SHA success↔success pair. "- "**Speedup uses success↔success pairs only** — mixed-conclusion pairs are counted in `Mixed` but excluded from the ratio. "- "`Fails` is the per-runner failure count across all observed runs (paired or not). Durations are seconds.\n"- )- print(- "| Workflow | Job | Success pairs | Depot median | Blacksmith median | Speedup (median) | Depot p95 | Blacksmith p95 | Depot fails | Blacksmith fails | Mixed |"- )- print("| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |")- summary_rows = []- stats_by_key: dict[tuple[str, str], dict] = {}- for (wf, base), by_sha in sorted(observations.items()):- s = derive_stats(by_sha)- stats_by_key[(wf, base)] = s- if not s["success_pairs"]:- continue # no comparable data — surfaced in singletons table below- depot = [p[0] for p in s["success_pairs"]]- bs = [p[1] for p in s["success_pairs"]]- dm = statistics.median(depot)- bm = statistics.median(bs)- # Renamed from `speedup` to avoid shadowing the str-typed `speedup` used- # in the CSV loop above — mypy unifies the type of repeated names in- # the same function scope and rejects the float here as `Any | float`- # vs `str`.- speedup_factor = dm / bm if bm > 0 else float("inf")- dp95 = sorted(depot)[max(0, int(len(depot) * 0.95) - 1)]- bp95 = sorted(bs)[max(0, int(len(bs) * 0.95) - 1)]- dfail = f"{s['depot_fail']}/{s['depot_total']}"- bfail = f"{s['bs_fail']}/{s['bs_total']}"- summary_rows.append(- (speedup_factor, wf, base, len(s["success_pairs"]), dm, bm, dp95, bp95, dfail, bfail, s["mixed_pairs"])- )-- # Sort by speedup descending — biggest wins at the top- for speedup_factor, wf, base, n, dm, bm, dp95, bp95, dfail, bfail, mixed in sorted(summary_rows, reverse=True):- print(- f"| `{wf}` | {base} | {n} | {dm:.0f}s | {bm:.0f}s | **{speedup_factor:.2f}x** | "- f"{dp95:.0f}s | {bp95:.0f}s | {dfail} | {bfail} | {mixed} |"- )-- # Unpaired / singleton jobs — highlights where the shadow didn't fire consistently- unpaired_rows = [- (wf, base, s["depot_only"], s["bs_only"])- for (wf, base), s in sorted(stats_by_key.items())- if s["depot_only"] or s["bs_only"]- ]- if unpaired_rows:- print("\n## Unpaired runs")- print(- "Jobs that ran on only one runner for a given SHA (shadow didn't fire, paths filter "- "differed, job was skipped on the other side, etc.). Large imbalances mean the "- "pairing denominator above is small.\n"- )- print("| Workflow | Job | Depot-only | Blacksmith-only |")- print("| --- | --- | ---: | ---: |")- for wf, base, d_only, b_only in unpaired_rows:- print(f"| `{wf}` | {base} | {d_only} | {b_only} |")-- print("\n## Interpretation")- print(- "- **Speedup > 1.0** → Blacksmith is faster on that job\n"- "- **Speedup < 1.0** → Blacksmith is slower (decision: is the price cut worth it?)\n"- "- **Fails (x/y)** → per-runner failure count across all observed runs. High asymmetry "- "(e.g. `5/50` vs `1/50`) points to runner-specific instability\n"- "- **Mixed** → pairs where one runner passed and the other failed. Excluded from speedup "- "but worth inspecting manually — the shadow may be masking a flake or a real break\n"- "- Low `Success pairs` count (< 10) → treat as noisy; widen window or re-run\n"- "- Compare `p95` to `median` — a large gap means runner variance is high\n"- )---if __name__ == "__main__":- main()diff --git a/.github/scripts/depot_backend_runs.py b/.github/scripts/depot_backend_runs.pyindex 497ce75d0a39..3840c0909f3a 100644--- a/.github/scripts/depot_backend_runs.py+++ b/.github/scripts/depot_backend_runs.py@@ -36,7 +36,6 @@ from pathlib import Pathfrom typing import TypedDict, castREPO = "PostHog/posthog"-# Both match .github/scripts/ci_backend_relay.py, which reads the same workflow for pull requests.WORKFLOW_NAME = "Backend CI on Depot"GATE_JOB_KEY = "ci-backend.yml:django_tests"GATE_CONCLUSIONS = {"finished": "success", "failed": "failure", "cancelled": "cancelled"}diff --git a/.github/scripts/test_ci_backend_depot_failures.py b/.github/scripts/test_ci_backend_depot_failures.pydeleted file mode 100644index 36b4c12d1595..000000000000--- a/.github/scripts/test_ci_backend_depot_failures.py+++ /dev/null@@ -1,49 +0,0 @@-import json-import subprocess-from typing import Any--import pytest--import ci_backend_depot_failures as depot_failures---def stub_diagnose(monkeypatch: pytest.MonkeyPatch, outcome: str | BaseException) -> None:- def run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess[str]:- if isinstance(outcome, BaseException):- raise outcome- return subprocess.CompletedProcess(args, 0, stdout=outcome)-- monkeypatch.setattr(depot_failures.subprocess, "run", run)---def test_log_lines_stay_inside_one_escaped_annotation(monkeypatch: pytest.MonkeyPatch) -> None:- lines = ["##[error]FAILED test_widget - 100% broken", "::add-mask::ghp_example", "tail\r\n::set-output name=x::y"]- attempt = {- "job_display_name": "Repo checks",- "error_message": "Step 3 (Validate widgets): exit 1",- "relevant_lines": [{"content": line} for line in lines],- }- stub_diagnose(monkeypatch, json.dumps({"representative_attempts": [attempt]}))-- assert depot_failures.explain("org", "workflow") == [- "::error title=Failed on Depot::Repo checks: Step 3 (Validate widgets): exit 1"- "%0AFAILED test_widget - 100%25 broken"- "%0A::add-mask::ghp_example"- "%0Atail%0D%0A::set-output name=x::y"- ]--- "outcome, error",- [- (FileNotFoundError("depot"), "FileNotFoundError"),- (subprocess.CalledProcessError(1, "depot"), "CalledProcessError"),- ("not json", "JSONDecodeError"),- ],-)-def test_unreadable_diagnosis_leaves_one_warning(- monkeypatch: pytest.MonkeyPatch, outcome: str | BaseException, error: str-) -> None:- stub_diagnose(monkeypatch, outcome)-- assert depot_failures.explain("org", "workflow") == [f"::warning::Cannot read the Depot failure details: {error}"]diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.pydeleted file mode 100644index 39d0c73185c4..000000000000--- a/.github/scripts/test_ci_backend_relay.py+++ /dev/null@@ -1,769 +0,0 @@-import json-import subprocess-import dataclasses-import http.client-import urllib.error-import urllib.parse-import importlib.util-import urllib.request-from collections.abc import Callable, Mapping, Sequence-from pathlib import Path-from typing import Any--import pytest--SCRIPT_PATH = Path(__file__).with_name("ci_backend_relay.py")-SPEC = importlib.util.spec_from_file_location("ci_backend_relay", SCRIPT_PATH)-assert SPEC is not None-assert SPEC.loader is not None-relay = importlib.util.module_from_spec(SPEC)-SPEC.loader.exec_module(relay)--PR = 105723-EVENT_AT = "2026-09-24T09:54:20Z"-EVENT = relay.Event(repo="PostHog/posthog", sha="a8a3755cf964", pr_number=PR, event_at=EVENT_AT)-EVENT_WAIT = relay.wait_check_name(PR, EVENT_AT)-PLAIN_WAIT = f"{relay.DEPOT_WORKFLOW} / {relay.WAIT_JOB}"-OLDER_RACING_WAIT = relay.wait_check_name(PR, "2026-09-24T09:54:19Z")-NEWER_RACING_WAIT = relay.wait_check_name(PR, "2026-09-24T09:54:22Z")---def api_run(id: int, state: str, workflow: str = "live", attempt: str = "") -> dict[str, Any]:- completed = state not in relay.PENDING_STATES- query = f"job=j1&attempt={attempt}" if attempt else "job=j1&repo=PostHog%2Fposthog"- return {- "id": id,- "status": "completed" if completed else state,- "conclusion": state if completed else None,- "app": {"id": relay.MIRROR_APP_ID if attempt else relay.DEPOT_APP_ID},- "details_url": f"https://depot.dev/orgs/ntsdt08fpt/workflows/{workflow}?{query}",- }---def run(id: int, state: str, workflow: str = "live", started_at: str = "2026-09-24T09:56:00Z") -> Any:- return relay.CheckRun.from_api({**api_run(id, state, workflow), "started_at": started_at})--- "event_waits,gates,expected",- [- pytest.param([], [], (relay.Phase.ABSENT, ""), id="no run yet"),- pytest.param([run(1, "in_progress")], [], (relay.Phase.STARTING, "in_progress"), id="waiting for hand-off"),- pytest.param([run(1, "failure")], [], (relay.Phase.DECLINED, "failure"), id="depot declined"),- pytest.param([run(1, "cancelled")], [], (relay.Phase.CANCELLED, "cancelled"), id="run cancelled"),- pytest.param(- [run(1, "success")],- [run(9, "cancelled", workflow="stale", started_at="2026-09-24T09:55:14Z")],- (relay.Phase.RUNNING, ""),- id="superseded run's cancelled gate is ignored",- ),- pytest.param(- [run(1, "success")],- [run(9, "cancelled", workflow="stale"), run(10, "success")],- (relay.Phase.FINISHED, "success"),- id="this run's gate wins over a newer id elsewhere",- ),- pytest.param(- [run(1, "cancelled", workflow="dup1"), run(2, "success", workflow="dup2")],- [run(10, "failure", workflow="dup2")],- (relay.Phase.FINISHED, "failure"),- id="duplicate run of the event replaces a cancelled one",- ),- pytest.param(- [run(1, "success")],- [run(10, "cancelled")],- (relay.Phase.CANCELLED, "cancelled"),- id="this run cancelled after the hand-off",- ),- pytest.param(- [run(1, "success")],- [run(10, "failure"), run(11, "success")],- (relay.Phase.FINISHED, "success"),- id="retried gate",- ),- ],-)-def test_progress_of_this_events_run(event_waits: list[Any], gates: list[Any], expected: tuple[Any, str]) -> None:- wait = relay.newest_live(event_waits)- result = relay.progress(wait, gates)- assert (result.phase, result.state) == expected---class FakeReader:- def __init__(self, polls: Sequence[dict[str, list[Any]]], advance_on: str = EVENT_WAIT) -> None:- self._polls = list(polls)- self._advance_on = advance_on- self.poll = 0- self.reads: list[str] = []-- def read(self, name: str) -> list[Any]:- self.reads.append(name)- if name == self._advance_on:- self.poll += 1- answer = self._polls[min(self.poll, len(self._polls)) - 1].get(name, [])- if isinstance(answer, Exception):- raise answer- return answer---class FakeClock:- def __init__(self) -> None:- self.now = 0.0-- def __call__(self) -> float:- return self.now-- def sleep(self, seconds: float) -> None:- self.now += seconds--- "polls,expected,min_minutes",- [- pytest.param(- [{EVENT_WAIT: [run(1, "cancelled", workflow="dup1")]}] * 3- + [{EVENT_WAIT: [run(1, "cancelled", workflow="dup1"), run(2, "success", workflow="dup2")]}]- + [- {- EVENT_WAIT: [run(1, "cancelled", workflow="dup1"), run(2, "success", workflow="dup2")],- relay.GATE_CHECK: [run(10, "success", workflow="dup2")],- }- ],- (relay.Phase.FINISHED, "success"),- 0,- id="a cancelled run is replaced within the grace window",- ),- pytest.param(- [{EVENT_WAIT: [run(1, "cancelled")]}],- (relay.Phase.CANCELLED, "cancelled"),- 15,- id="a cancelled run fails only after the grace window",- ),- pytest.param([{}], (relay.Phase.ABSENT, ""), 15, id="no run fails after the grace window"),- pytest.param(- [{EVENT_WAIT: [run(1, "success")]}] * 16- + [{EVENT_WAIT: relay.ReadFailedError("Cannot read")}]- + [{EVENT_WAIT: [run(1, "success")], relay.GATE_CHECK: [run(10, "success")]}],- (relay.Phase.FINISHED, "success"),- 16,- id="a failed read after the grace window keeps waiting",- ),- pytest.param(- [- {- OLDER_RACING_WAIT: [run(1, "success", workflow="racing")],- relay.GATE_CHECK: [run(10, "failure", workflow="racing")],- }- ],- (relay.Phase.FINISHED, "failure"),- 15,- id="an older racing event's run stands in for an absent run",- ),- pytest.param(- [- {- EVENT_WAIT: [run(1, "cancelled")],- NEWER_RACING_WAIT: [run(2, "success", workflow="racing")],- relay.GATE_CHECK: [run(10, "success", workflow="racing")],- }- ],- (relay.Phase.FINISHED, "success"),- 15,- id="a newer racing event's run stands in for a cancelled run",- ),- pytest.param(- [- {- NEWER_RACING_WAIT: [run(1, "success", workflow="newer")],- OLDER_RACING_WAIT: [run(2, "success", workflow="older")],- relay.GATE_CHECK: [run(10, "cancelled", workflow="newer"), run(11, "success", workflow="older")],- }- ],- (relay.Phase.FINISHED, "success"),- 15,- id="the next racing event's run stands in when the followed one is cancelled",- ),- pytest.param(- [- {- NEWER_RACING_WAIT: [run(1, "success", workflow="newer")],- OLDER_RACING_WAIT: [run(2, "success", workflow="older")],- relay.GATE_CHECK: [run(10, "success", workflow="newer"), run(11, "failure", workflow="older")],- }- ],- (relay.Phase.FINISHED, "success"),- 15,- id="the newer of two racing runs stands in",- ),- pytest.param(- [- {- NEWER_RACING_WAIT: [run(1, "skipped", workflow="declined")],- OLDER_RACING_WAIT: [run(2, "success", workflow="older")],- relay.GATE_CHECK: [run(10, "success", workflow="older")],- }- ],- (relay.Phase.FINISHED, "success"),- 15,- id="a racing run that declined the hand-off does not stand in",- ),- pytest.param(- [- {- relay.wait_check_name(PR, "2026-09-24T09:54:17Z"): [run(1, "success", workflow="earlier")],- relay.GATE_CHECK: [run(10, "success", workflow="earlier")],- }- ],- (relay.Phase.ABSENT, ""),- 15,- id="an event outside the race window does not stand in",- ),- pytest.param(- [{EVENT_WAIT: [run(1, "success")]}],- (relay.Phase.RUNNING, ""),- 90,- id="a running gate hits the overall deadline",- ),- ],-)-def test_poll_waits_out_a_replacement_before_failing(- polls: list[dict[str, list[Any]]], expected: tuple[Any, str], min_minutes: int-) -> None:- clock = FakeClock()- result = relay.poll(- FakeReader(polls),- EVENT,- relay.GATE_CHECK,- deadline_minutes=90,- absent_minutes=15,- clock=clock,- sleep=clock.sleep,- )- assert (result.phase, result.state) == expected- assert clock.now >= min_minutes * 60---def test_poll_does_not_use_an_ambiguous_plain_named_wait() -> None:- clock = FakeClock()- reader = FakeReader([{PLAIN_WAIT: [run(1, "success")], relay.GATE_CHECK: [run(2, "success")]}])- result = relay.poll(- reader, EVENT, relay.GATE_CHECK, deadline_minutes=90, absent_minutes=15, clock=clock, sleep=clock.sleep- )- assert result.phase == relay.Phase.ABSENT- assert PLAIN_WAIT not in reader.reads--- "result,exit_code,first_line",- [- (relay.Progress(relay.Phase.FINISHED, "success"), 0, None),- (- relay.Progress(relay.Phase.FINISHED, "failure", "https://depot.dev/orgs/o/workflows/w1?job=j"),- 1,- "concluded failure",- ),- (relay.Progress(relay.Phase.CANCELLED, "cancelled"), 1, "cancelled its run"),- (relay.Progress(relay.Phase.DECLINED, "failure"), 1, "declined the hand-off"),- (relay.Progress(relay.Phase.ABSENT), 1, "started no run"),- (relay.Progress(relay.Phase.RUNNING), 1, "No Depot verdict"),- ],-)-def test_relay_gate_fails_closed(result: Any, exit_code: int, first_line: str | None) -> None:- code, lines = relay.relay_gate(result, EVENT, "123")- assert code == exit_code- if first_line is None:- assert lines == []- else:- assert first_line in lines[0]--- "merge_queue,present,absent",- [- (False, "--add-label ci-backend-github", "CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT"),- (True, "CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT", "--add-label ci-backend-github"),- ],-)-def test_retry_instructions_route_back_to_github_the_way_that_works(- merge_queue: bool, present: str, absent: str-) -> None:- event = dataclasses.replace(EVENT, merge_queue=merge_queue)- text = "\n".join(relay.retry_instructions(event, "https://depot.dev/orgs/o/workflows/w1", "123"))- assert present in text- assert absent not in text---def shown(workflow: str, gate: str, executions: int) -> dict[str, Any]:- return {- "workflow": {"status": workflow},- "run": {"run_id": "r1"},- "executions": [{}] * executions,- "jobs": [- {"job_key": "ci-backend.yml:changes", "status": "finished"},- {"job_key": relay.GATE_JOB_KEY, "status": gate},- ],- }---class FakeDepot:- def __init__(self, shows: Sequence[Any]) -> None:- self._shows = list(shows)- self._last: dict[str, Any] = {}- self.sent: list[tuple[str, ...]] = []-- def __call__(self, *args: str) -> Any:- if args[0] != "workflow":- # Depot refuses a retry of a workflow that still runs.- assert self._last["workflow"]["status"] not in relay.DEPOT_LIVE_STATES- self.sent.append(args)- if self._last.get("refuses_retry"):- raise subprocess.CalledProcessError(1, "depot")- return {}- answer = self._shows.pop(0) if len(self._shows) > 1 else self._shows[0]- if isinstance(answer, Exception):- raise answer- self._last = answer- return answer---RETRY_FAILED = ("retry", "r1", "--workflow", "w1", "--org", "o1", "--failed")--- "shows,expected,sent",- [- pytest.param(- [- shown("running", "failed", 1),- shown("failed", "failed", 1),- shown("failed", "failed", 1),- shown("running", "queued", 2),- shown("failed", "failed", 2),- ],- (relay.Phase.FINISHED, "failure"),- [RETRY_FAILED],- id="one retry, sent after the workflow finished, and the earlier verdict is not relayed",- ),- pytest.param(- [shown("running", "queued", 2), shown("running", "finished", 2)],- (relay.Phase.FINISHED, "success"),- [],- id="a gate that another retry turned green needs no retry",- ),- pytest.param(- [shown("failed", "failed", 1), shown("running", "queued", 2)],- (relay.Phase.RUNNING, ""),- [RETRY_FAILED],- id="a retry that never finishes hits the deadline",- ),- pytest.param([OSError("no depot CLI")], None, [], id="an unreachable Depot gives no verdict"),- pytest.param(- [{**shown("failed", "failed", 1), "refuses_retry": True}],- None,- [RETRY_FAILED],- id="a refused retry is not sent again",- ),- ],-)-def test_retry_relays_only_the_verdict_of_the_new_execution(- shows: list[Any], expected: tuple[Any, str] | None, sent: list[tuple[str, ...]]-) -> None:- depot = FakeDepot(shows)- clock = FakeClock()- result = relay.retry_failed_jobs("o1", "w1", depot=depot, clock=clock, sleep=clock.sleep)- assert (result and (result.phase, result.state)) == expected- assert depot.sent == sent--- "rerun,polls,prerequisite,retry_answer,expected,retries",- [- pytest.param(False, [run(10, "failure")], [], "success", "failure", 0, id="a first attempt retries nothing"),- pytest.param(True, [run(10, "failure")], [], "success", "success", 1, id="a failed gate is retried"),- pytest.param(True, [run(10, "success")], [], "failure", "success", 0, id="a passed gate is relayed"),- pytest.param(True, [run(10, "failure")], [], None, "failure", 1, id="Depot unreachable"),- pytest.param(True, [run(10, "cancelled")], [], "success", "success", 1, id="a cancelled gate is retried"),- pytest.param(True, [None, run(10, "success")], [], "failure", "success", 0, id="a running gate is awaited"),- pytest.param(- True,- [run(10, "cancelled")],- [run(20, "failure")],- "success",- "failure",- 0,- id="a failed prerequisite is not retried",- ),- ],-)-def test_only_a_rerun_with_a_failed_gate_retries_on_depot(- rerun: bool, polls: list[Any], prerequisite: list[Any], retry_answer: str | None, expected: str, retries: int-) -> None:- calls: list[tuple[str, str]] = []-- def retry(org: str, workflow: str) -> Any:- calls.append((org, workflow))- return retry_answer and relay.Progress(relay.Phase.FINISHED, retry_answer)-- reader = FakeReader(- [- {- EVENT_WAIT: [run(1, "success")],- relay.GATE_CHECK: [gate] if gate else [],- f"{relay.DEPOT_WORKFLOW} / {relay.PREREQUISITES[0]}": prerequisite,- }- for gate in polls- ]- )- result = relay.gate_verdict(reader, EVENT, rerun=rerun, retry=retry)- assert result.state == expected- assert calls == [(relay.DEPOT_ORG, "live")] * retries---def api_check() -> dict[str, Any]:- return {- "id": 1,- "status": "completed",- "conclusion": "success",- "app": {"id": relay.DEPOT_APP_ID},- "name": relay.GATE_CHECK,- "head_sha": "abc",- "pull_requests": [{"number": PR}],- "details_url": "https://depot.dev/orgs/ntsdt08fpt/workflows/live?job=j1&repo=PostHog%2Fposthog",- }---class FakeResponse:- def __init__(self, body: bytes, etag: str) -> None:- self.status = 200- self.headers = {"ETag": etag}- self._body = body-- def read(self, limit: int = -1) -> bytes:- return self._body-- def __enter__(self) -> "FakeResponse":- return self-- def __exit__(self, *_: object) -> None:- return None---def checks_opener(- checks: Mapping[int, Mapping[str, Sequence[dict[str, Any]] | None]],-) -> Callable[[urllib.request.Request, int], FakeResponse]:- def opener(request: urllib.request.Request, timeout: int) -> FakeResponse:- query = urllib.parse.parse_qs(urllib.parse.urlparse(request.full_url).query)- name = query["check_name"][0]- runs = checks[int(query["app_id"][0])][name]- if runs is None:- raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) # type: ignore[arg-type]- body = [{**check, "name": name, "head_sha": EVENT.sha} for check in runs]- return FakeResponse(json.dumps({"check_runs": body}).encode(), "")-- return opener---def test_reader_reuses_its_answer_on_304_and_stops_on_repeated_refusals() -> None:- sent: list[dict[str, str]] = []- answers: list[Any] = [- FakeResponse(json.dumps({"check_runs": [api_check()]}).encode(), '"e1"'),- urllib.error.HTTPError("url", 304, "Not Modified", {}, None), # type: ignore[arg-type]- *[urllib.error.HTTPError("url", 403, "Forbidden", {}, None) for _ in range(relay.MAX_REFUSALS)], # type: ignore[arg-type]- ]-- def opener(request: Any, timeout: int) -> Any:- sent.append(dict(request.header_items()))- answer = answers.pop(0)- if isinstance(answer, Exception):- raise answer- return answer-- reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,))- assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"]- assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"]- assert sent[1]["If-none-match"] == '"e1"'- for _ in range(relay.MAX_REFUSALS - 1):- with pytest.raises(relay.ReadFailedError):- reader.read(relay.GATE_CHECK)- with pytest.raises(relay.ReadRefusedError):- reader.read(relay.GATE_CHECK)--[email protected]("page", [1, 2])- "error",- [ConnectionResetError("reset"), http.client.IncompleteRead(b""), ValueError("invalid JSON")],-)-def test_reader_retries_interrupted_pages_without_reusing_a_stale_verdict(page: int, error: Exception) -> None:- payload = api_check()- answers: list[Any] = [FakeResponse(json.dumps({"check_runs": [payload]}).encode(), '"e1"')]- if page == 2:- answers.append(FakeResponse(json.dumps({"check_runs": [payload] * relay.PAGE_SIZE}).encode(), '"e2"'))- answers += [error, FakeResponse(b'{"check_runs": []}', '"e3"')]-- def opener(request: Any, timeout: int) -> FakeResponse:- answer = answers.pop(0)- if isinstance(answer, Exception):- raise answer- return answer-- reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,))- assert reader.read(relay.GATE_CHECK)[0].state == "success"- with pytest.raises(relay.ReadFailedError):- reader.read(relay.GATE_CHECK)- assert reader.read(relay.GATE_CHECK) == []- assert not answers--[email protected]("name", relay.PREREQUISITES)- "state,workflow,newer",- [- ("failure", "live", None),- ("failure", "other", None),- ("success", "live", None),- ("failure", "live", "pending"),- ("failure", "live", "success"),- ],-)-def test_cancelled_gate_reports_only_current_selected_prerequisite(- name: str, state: str, workflow: str, newer: str | None-) -> None:- roots = [run(2, state, workflow)]- if newer:- roots.append(run(3, newer, workflow))- clock = FakeClock()- result = relay.poll(- FakeReader(- [- {- EVENT_WAIT: [run(1, "success")],- relay.GATE_CHECK: [run(5, "cancelled")],- f"{relay.DEPOT_WORKFLOW} / {name}": roots,- }- ]- ),- EVENT,- relay.GATE_CHECK,- deadline_minutes=90,- absent_minutes=15,- clock=clock,- sleep=clock.sleep,- )- code, lines = relay.relay_gate(result, EVENT, "123")- assert code == 1- if state == "failure" and workflow == "live" and newer is None:- assert clock.now == 0- assert lines == [f"::error::{name} failed on Depot (check 2). Push a fix; a retry will not help."]- assert result.root_check_id == 2- else:- assert result.phase == relay.Phase.CANCELLED- assert clock.now == 900--[email protected]("name", relay.PREREQUISITES)-def test_failed_gate_keeps_retry_options_after_a_prerequisite_failure(name: str) -> None:- # Without Depot's self-cancel, the prerequisite may have failed on a retryable setup step.- clock = FakeClock()- result = relay.poll(- FakeReader(- [- {- EVENT_WAIT: [run(1, "success")],- relay.GATE_CHECK: [run(5, "failure")],- f"{relay.DEPOT_WORKFLOW} / {name}": [run(2, "failure")],- }- ]- ),- EVENT,- relay.GATE_CHECK,- deadline_minutes=90,- absent_minutes=15,- clock=clock,- sleep=clock.sleep,- )- code, lines = relay.relay_gate(result, EVENT, "123")- assert code == 1- assert not result.root_failure- assert not any("a retry will not help" in line for line in lines)--- "field,value",- [- ("app", {"id": 99}),- ("app", None),- ("name", "other check"),- ("head_sha", "other"),- ("pull_requests", [{"number": PR + 1}]),- ("details_url", "https://depot.dev/orgs/other/workflows/live"),- ("details_url", "https://depot.dev.example.com/orgs/ntsdt08fpt/workflows/live"),- ("details_url", "https://[email protected]/orgs/ntsdt08fpt/workflows/live"),- ("details_url", "http://depot.dev/orgs/ntsdt08fpt/workflows/live"),- ],-)-def test_reader_rejects_wrong_identity(field: str, value: Any) -> None:- payload = {**api_check(), field: value}- reader = relay.CheckRunReader(- "PostHog/posthog",- "abc",- "token",- pr_number=PR,- opener=lambda *a, **kw: FakeResponse(json.dumps({"check_runs": [payload]}).encode(), ""),- )- assert reader.read(relay.GATE_CHECK) == []---def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None:- answer = json.dumps({"check_runs": [{**api_check(), "app": None}, api_check()]}).encode()- reader = relay.CheckRunReader(- "PostHog/posthog", "abc", "token", pr_number=PR, opener=lambda *a, **kw: FakeResponse(answer, "")- )- assert len(reader.read(relay.GATE_CHECK)) == 1--- "mirror_gate,depot_gate,expected",- [- pytest.param(- [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")],- [],- (relay.Phase.FINISHED, "success"),- id="Depot's copies lag",- ),- pytest.param(- [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")],- [(9, "failure", "w1")],- (relay.Phase.FINISHED, "success"),- id="a late copy of an older attempt",- ),- pytest.param(- [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")],- [(8, "success", "w1"), (9, "failure", "w1")],- (relay.Phase.FINISHED, "success"),- id="late copies of both attempts",- ),- pytest.param(- [(2, "failure", "w1", "a1"), (5, "failure", "w1", "a2")],- [(4, "failure", "w1"), (9, "failure", "w1")],- (relay.Phase.FINISHED, "failure"),- id="a retry that failed again",- ),- pytest.param(- [(2, "success", "w1", "a1")],- [(4, "success", "w1"), (9, "failure", "w1")],- (relay.Phase.FINISHED, "failure"),- id="a newer attempt the mirror missed",- ),- pytest.param(- [(2, "failure", "w1", "a1")],- [(4, "failure", "w1"), (9, "in_progress", "w1")],- (relay.Phase.RUNNING, "in_progress"),- id="a running retry the mirror has not posted",- ),- pytest.param(- [(5, "success", "w1", "a2")],- [(4, "cancelled", "w1")],- (relay.Phase.FINISHED, "success"),- id="a replacement for a job cancelled before it started",- ),- pytest.param(- [(5, "success", "w1", "a2")],- [(4, "cancelled", "w1"), (9, "in_progress", "w1")],- (relay.Phase.RUNNING, "in_progress"),- id="a replacement Depot has not finished after a cancelled job",- ),- pytest.param(- [(5, "success", "other", "a1")],- [(9, "failure", "w1")],- (relay.Phase.FINISHED, "failure"),- id="another workflow's mirrored gate",- ),- pytest.param(- [(5, "success", "other", "a1")],- [],- (relay.Phase.RUNNING, ""),- id="only another workflow's mirrored gate",- ),- pytest.param([(5, "success", "w1", "a1")], None, None, id="Depot's app read fails"),- ],-)-def test_relay_reads_the_current_attempt_across_apps(- mirror_gate: list[tuple[int, str, str, str]],- depot_gate: list[tuple[int, str, str]] | None,- expected: tuple[Any, str] | None,-) -> None:- mirror = {- EVENT_WAIT: [api_run(1, "success", "w1", "w")],- relay.GATE_CHECK: [],- relay.MIRRORED_GATE_CHECK: [api_run(*c) for c in mirror_gate],- }- depot = {- EVENT_WAIT: [api_run(3, "success", "w1")],- relay.GATE_CHECK: None if depot_gate is None else [api_run(*c) for c in depot_gate],- }-- reader = relay.CheckRunReader(- EVENT.repo, EVENT.sha, "token", opener=checks_opener({relay.MIRROR_APP_ID: mirror, relay.DEPOT_APP_ID: depot})- )- wait = relay.newest_live(reader.read(EVENT_WAIT))- if expected is None:- with pytest.raises(relay.ReadFailedError):- reader.read(relay.GATE_CHECK)- return- current = relay.progress(wait, reader.read(relay.GATE_CHECK))- assert (current.phase, current.state) == expected--- "posted,unreadable,expected",- [- pytest.param([(relay.GATE_CHECK, "success")], None, "success", id="an older workflow revision"),- pytest.param(- [(relay.GATE_CHECK, "success")],- relay.MIRRORED_GATE_CHECK,- None,- id="a failed read of the mirror's new name",- ),- pytest.param(- [(relay.MIRRORED_GATE_CHECK, "success")],- relay.GATE_CHECK,- None,- id="a failed read of the mirror's old name",- ),- pytest.param(- [(relay.GATE_CHECK, "success"), (relay.MIRRORED_GATE_CHECK, "failure")],- None,- "failure",- id="a newer failed attempt under the new name",- ),- pytest.param(- [(relay.MIRRORED_GATE_CHECK, "success"), (relay.GATE_CHECK, "failure")],- None,- "failure",- id="a newer failed attempt under the old name",- ),- ],-)-def test_reader_reads_the_mirrored_gate_under_both_names(- posted: Sequence[tuple[str, str]], unreadable: str | None, expected: str | None-) -> None:- mirror: dict[str, list[dict[str, Any]] | None] = {relay.GATE_CHECK: [], relay.MIRRORED_GATE_CHECK: []}- for index, (name, state) in enumerate(posted, start=1):- mirror[name] = [api_run(index, state, "w1", f"a{index}")]- if unreadable:- mirror[unreadable] = None- reader = relay.CheckRunReader(- EVENT.repo,- EVENT.sha,- "token",- opener=checks_opener({relay.MIRROR_APP_ID: mirror}),- app_ids=(relay.MIRROR_APP_ID,),- )- if expected is None:- with pytest.raises(relay.ReadFailedError):- reader.read(relay.GATE_CHECK)- return- current = relay.current_check(reader.read(relay.GATE_CHECK), "w1")- assert current is not None- assert current.state == expecteddiff --git a/.github/scripts/test_ci_backend_route.py b/.github/scripts/test_ci_backend_route.pydeleted file mode 100644index 24c2c8a28f46..000000000000--- a/.github/scripts/test_ci_backend_route.py+++ /dev/null@@ -1,291 +0,0 @@-import json-import urllib.error-import importlib.util-from collections.abc import Sequence-from datetime import UTC, datetime-from pathlib import Path-from typing import Any--import pytest--SCRIPT_PATH = Path(__file__).with_name("ci_backend_route.py")-SPEC = importlib.util.spec_from_file_location("ci_backend_route", SCRIPT_PATH)-assert SPEC is not None-assert SPEC.loader is not None-route = importlib.util.module_from_spec(SPEC)-SPEC.loader.exec_module(route)--EVENT_AT = "2026-09-30T05:44:10Z"-EVENT_EPOCH = datetime.strptime(EVENT_AT, route.EVENT_TIME).replace(tzinfo=UTC).timestamp()---def pr(- percent: int = 25, number: int | None = 125, labels: Sequence[str] = (), fork: bool = False, draft: bool = False-) -> Any:- return route.decide("pull_request", percent, number, list(labels), fork, draft)--[email protected]("number,bucket", [(0, 40), (7, 30), (124, 94), (125, 19), (103100, 33)])-def test_bucket_is_a_fixed_hash_of_the_pr_number(number: int, bucket: int) -> None:- assert route.bucket_of(number) == bucket- assert pr(number=number, percent=bucket).engine == "github"- assert pr(number=number, percent=bucket + 1).engine == "depot"---def test_consecutive_prs_spread_across_buckets() -> None:- routed = [pr(number=number, percent=5).engine == "depot" for number in range(100000, 101000)]- assert 30 <= sum(routed) <= 70- assert not any(all(routed[i : i + 5]) for i in range(len(routed) - 4))--- "labels,fork,draft,expected",- [- (["ci-backend-github"], False, False, "github"),- (["ci-backend-depot"], False, False, "depot"),- (["ci-backend-depot", "ci-backend-github"], False, False, "github"),- (["ci-backend-depot"], True, False, "github"),- (["ci-backend-depot", "no-ci"], False, True, "github"),- (["no-ci"], False, False, "depot"),- ],-)-def test_overrides(labels: list[str], fork: bool, draft: bool, expected: str) -> None:- assert pr(labels=labels, fork=fork, draft=draft).engine == expected--[email protected]("event", ["workflow_dispatch", "push", "schedule", "merge_group"])-def test_non_pull_request_events_stay_on_github(event: str) -> None:- assert route.decide(event, 100, None, ["ci-backend-depot"], False, False, "success").engine == "github"--- "head_ref,labels,percent,queue_percent,prior,expected",- [- ("trunk-merge/pr-1/x", ["ci-backend-depot"], 100, 0, None, "github"),- ("trunk-merge/pr-1/x", ["ci-backend-github"], 0, 100, None, "depot"),- ("trunk-merge/pr-1/x", [], 0, 0, "success", "depot"),- ("feature/trunk-merge", [], 100, 0, None, "depot"),- ],-)-def test_merge_queue_batches_route_by_their_own_percent(- head_ref: str, labels: list[str], percent: int, queue_percent: int, prior: str | None, expected: str-) -> None:- decision = route.decide("pull_request", percent, 124, labels, False, True, prior, head_ref, queue_percent)- assert decision.engine == expected---def test_missing_pr_number_stays_on_github() -> None:- assert pr(number=None, percent=100).engine == "github"--- "prior,labels,percent,expected",- [- ("success", ["ci-backend-github"], 0, "depot"),- ("skipped", ["ci-backend-depot"], 100, "github"),- ("cancelled", ["ci-backend-depot"], 0, "depot"),- (None, [], 100, "depot"),- ],-)-def test_earlier_run_of_the_commit_wins(prior: str | None, labels: list[str], percent: int, expected: str) -> None:- assert route.decide("pull_request", percent, 124, labels, False, False, prior).engine == expected---def check(run_id: int, status: str, conclusion: str | None, pr_number: int = 124) -> dict[str, Any]:- return {"id": run_id, "status": status, "conclusion": conclusion, "pull_requests": [{"number": pr_number}]}--- "check_runs,expected",- [- ([check(1, "completed", "success"), check(2, "queued", None)], "success"),- ([check(1, "completed", "success"), check(2, "completed", "cancelled")], "success"),- ([check(1, "completed", "success"), check(2, "completed", "skipped")], "success"),- ([check(1, "completed", "skipped"), check(2, "completed", "cancelled")], "skipped"),- ([check(1, "completed", "cancelled"), check(2, "completed", "failure")], "failure"),- ([check(1, "completed", "success", pr_number=125)], None),- ([{"id": 1, "status": "completed", "conclusion": "success", "pull_requests": []}], None),- ([], None),- ],-)-def test_handoff_conclusion_keeps_the_engine_that_already_ran_this_pull_request(- check_runs: list[dict[str, Any]], expected: str | None-) -> None:- assert route.handoff_conclusion(check_runs, 124) == expected---class FakeResponse:- def __init__(self, payload: Any) -> None:- self.payload = payload-- def read(self) -> bytes:- return json.dumps(self.payload).encode()-- def __enter__(self) -> "FakeResponse":- return self-- def __exit__(self, *exc: object) -> None:- return None---def http_error(code: int) -> urllib.error.HTTPError:- return urllib.error.HTTPError("https://api.github.com", code, "boom", {}, None) # type: ignore[arg-type]---def opener_from(answers: list[Any]) -> Any:- def opener(request: Any, timeout: int) -> Any:- answer = answers.pop(0)- if isinstance(answer, Exception):- raise answer- return FakeResponse(answer)-- return opener---def test_fetch_retries_a_server_error_then_returns_the_checks(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(route.time, "sleep", lambda seconds: None)- checks = [check(1, "completed", "success")]- opener = opener_from([http_error(502), {"check_runs": checks}])- assert route.fetch_handoff_checks("PostHog/posthog", "abc", "t", opener=opener) == checks--- "answers,attempts_used",- [([http_error(403)], 1), ([http_error(503)] * 3, 3), ([urllib.error.URLError("down")] * 3, 3)],-)-def test_fetch_fails_closed_without_retrying_client_errors(- answers: list[Exception], attempts_used: int, monkeypatch: pytest.MonkeyPatch-) -> None:- monkeypatch.setattr(route.time, "sleep", lambda seconds: None)- remaining = list(answers)- with pytest.raises(route.HandoffReadError):- route.fetch_handoff_checks("PostHog/posthog", "abc", "t", opener=opener_from(remaining))- assert len(answers) - len(remaining) == attempts_used--- "raw,expected",- [(None, 0), ("", 0), ("abc", 0), ("-5", 0), ("42", 42), ("250", 100), (" 7 ", 7), ("²", 0), ("9" * 5000, 0)],-)-def test_parse_percent_fails_closed(raw: str | None, expected: int) -> None:- assert route.parse_percent(raw) == expected--- "labels,started,head_ref,expected",- [- (json.dumps(["other"]), True, "", "engine=depot\nreason=bucket 30 < 50%\n"),- ("null", True, "", "engine=depot\nreason=bucket 30 < 50%\n"),- ("", True, "", "engine=depot\nreason=bucket 30 < 50%\n"),- ("[]", False, "", "engine=github\nreason=Depot CI started no run for this event\n"),- (json.dumps(["ci-backend-depot"]), False, "", "engine=github\nreason=Depot CI started no run for this event\n"),- ("[]", True, "trunk-merge/pr-1/x", "engine=github\nreason=merge queue bucket 30 >= 20%\n"),- ],-)-def test_main_writes_outputs(- labels: str, started: bool, head_ref: str, expected: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch-) -> None:- output = tmp_path / "out"- monkeypatch.setenv("GITHUB_OUTPUT", str(output))- monkeypatch.setenv("EVENT", "pull_request")- monkeypatch.setenv("PERCENT", "50")- monkeypatch.setenv("MERGE_QUEUE_PERCENT", "20")- monkeypatch.setenv("HEAD_REF", head_ref)- monkeypatch.setenv("PR_NUMBER", "7")- monkeypatch.setenv("LABELS", labels)- monkeypatch.setattr(route, "fetch_handoff_checks", lambda repo, sha, token: [])- monkeypatch.setattr(route, "depot_started", lambda *_: started)- monkeypatch.setenv("REPO", "PostHog/posthog")- monkeypatch.setenv("SHA", "abc")- monkeypatch.setenv("GH_TOKEN", "t")- assert route.main() == 0- assert output.read_text() == expected--[email protected]("percent", ["", "0", "5"])-def test_main_keeps_a_handed_off_commit_on_depot_after_rollback(- percent: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch-) -> None:- def fetch(repo: str, sha: str, token: str) -> list[dict[str, Any]]:- return [{"id": 1, "status": "completed", "conclusion": "success", "pull_requests": [{"number": 7}]}]-- output = tmp_path / "out"- monkeypatch.setenv("GITHUB_OUTPUT", str(output))- monkeypatch.setenv("EVENT", "pull_request")- monkeypatch.setenv("PERCENT", percent)- monkeypatch.setenv("PR_NUMBER", "7")- monkeypatch.setenv("LABELS", "[]")- monkeypatch.setenv("REPO", "PostHog/posthog")- monkeypatch.setenv("SHA", "abc")- monkeypatch.setenv("GH_TOKEN", "t")- monkeypatch.setattr(route, "fetch_handoff_checks", fetch)- monkeypatch.setattr(route, "depot_started", lambda *_: False)- assert route.main() == 0- assert output.read_text().startswith("engine=depot\n")--- "percent,labels",- [- ("50", "[]"),- ("", json.dumps(["ci-backend-depot"])),- ("5", "[]"),- ("0", "[]"),- ("", "[]"),- ("50", json.dumps(["ci-backend-github"])),- ],-)-def test_main_never_switches_engines_on_an_unreadable_handoff(- percent: str, labels: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch-) -> None:- def failing(repo: str, sha: str, token: str) -> list[dict[str, Any]]:- raise route.HandoffReadError("boom")-- output = tmp_path / "out"- monkeypatch.setenv("GITHUB_OUTPUT", str(output))- monkeypatch.setenv("EVENT", "pull_request")- monkeypatch.setenv("PERCENT", percent)- monkeypatch.setenv("PR_NUMBER", "7")- monkeypatch.setenv("LABELS", labels)- monkeypatch.setenv("REPO", "PostHog/posthog")- monkeypatch.setenv("SHA", "abc")- monkeypatch.setenv("GH_TOKEN", "t")- monkeypatch.setattr(route, "fetch_handoff_checks", failing)- monkeypatch.setattr(route.time, "sleep", lambda seconds: None)- assert route.main() == 1- assert not output.exists()---class FakeReader:- def __init__(self, polls: list[bool | Exception]) -> None:- self.polls = polls-- def read(self, name: str) -> list[str]:- assert name == f"Backend CI on Depot / Depot run started (PR 7, event {EVENT_AT})"- answer = self.polls.pop(0) if len(self.polls) > 1 else self.polls[0]- if isinstance(answer, Exception):- raise answer- return ["check"] if answer else []--- "polls,started,routed_after",- [- ([True], True, 15),- ([False, False, True], True, 15),- ([route.ReadFailedError("boom"), True], True, 15),- ([False], False, 15),- ([route.ReadRefusedError("refused")], False, 15),- ([False], False, 400),- ],-)-def test_depot_started_waits_for_the_started_check_of_the_event(- polls: list[bool | Exception], started: bool, routed_after: int-) -> None:- now = [EVENT_EPOCH + routed_after]-- def sleep(seconds: float) -> None:- now[0] += seconds-- assert route.depot_started(FakeReader(polls), 7, EVENT_AT, clock=lambda: now[0], sleep=sleep) is started- assert now[0] <= max(EVENT_EPOCH + route.DEPOT_START_SECONDS + route.DEPOT_POLL_SECONDS, EVENT_EPOCH + routed_after)diff --git a/.github/scripts/test_ci_flake_overseer.py b/.github/scripts/test_ci_flake_overseer.pydeleted file mode 100644index fc3ad7e0de53..000000000000--- a/.github/scripts/test_ci_flake_overseer.py+++ /dev/null@@ -1,359 +0,0 @@-from __future__ import annotations--import json--import pytest--import ci_flake_overseer-from ci_flake_overseer import (- DECISION_EVENT,- OUTCOME_EVENT,- Decision,- Job,- Step,- WorkflowRun,- build_decision_events,- capture_events,- classify_job,- is_test_job_failure,- report_rerun_outcomes,- rerun_outcome_label,- workflow_run_from_object,-)---def make_job(- name: str,- *,- run_attempt: int = 1,- conclusion: str = "failure",- failed_step: str = "Run Core tests",- started_at: str = "2026-06-05T21:00:00Z",-) -> Job:- return Job(- id=123,- name=name,- conclusion=conclusion,- run_attempt=run_attempt,- html_url="https://github.com/PostHog/posthog/actions/runs/1/job/123",- started_at=started_at,- steps=(Step(name=failed_step, conclusion="failure"),),- )---def make_workflow_run(- run_attempt: int = 1, *, head_branch: str = "", event: str = "pull_request", pr_number: int | None = 12345-) -> WorkflowRun:- return WorkflowRun(- id=999,- workflow_id=42,- name="Backend CI",- conclusion="failure",- head_sha="abc123",- run_attempt=run_attempt,- html_url="https://github.com/PostHog/posthog/actions/runs/999",- head_branch=head_branch,- event=event,- pr_number=pr_number,- )--- ("job", "expected_action", "expected_reason_fragment", "expected_failure_class"),- [- pytest.param(- make_job("Repo checks (depot-ubuntu-latest)", failed_step="Check module boundaries (tach)"),- "skip deterministic",- "deterministic",- "repo-checks",- id="deterministic-repo-checks",- ),- pytest.param(- make_job("Validate OpenAPI types", failed_step="Check and update OpenAPI types"),- "skip deterministic",- "deterministic",- "openapi",- id="deterministic-openapi",- ),- pytest.param(- make_job("Frontend lint", failed_step="Lint with Oxlint"),- "skip deterministic",- "deterministic",- "lint",- id="deterministic-lint",- ),- pytest.param(- make_job("Validate migrations", failed_step="Check migrations"),- "skip deterministic",- "deterministic",- "migrations",- id="deterministic-migrations",- ),- pytest.param(- make_job("Django tests", failed_step="Product facade enforcement"),- "skip deterministic",- "deterministic",- "module-boundaries",- id="deterministic-via-step-rule",- ),- pytest.param(- make_job("Build and deploy", failed_step="Compile assets"),- "skip non-test",- "not an allowlisted test runner",- None,- id="non-test-job",- ),- pytest.param(- make_job("Django tests - Temporal (1/1)", failed_step="Run Temporal tests"),- "observe",- "test job failure",- None,- id="observed-test-job",- ),- ],-)-def test_classify_job(- job: Job, expected_action: str, expected_reason_fragment: str, expected_failure_class: str | None-) -> None:- decision = classify_job(job)-- assert decision.action == expected_action- assert expected_reason_fragment in decision.reason- assert decision.failure_class == expected_failure_class--- ("job_name", "failed_step", "expected"),- [- pytest.param("Django tests - Temporal (1/1)", "Run Core tests", True, id="test-job-and-test-step"),- pytest.param(- "Django tests - Temporal (1/1)", "Stop containers", True, id="test-job-nontest-step-falls-back-to-name"- ),- pytest.param("Build and deploy", "Run Playwright tests", True, id="nontest-job-test-step"),- pytest.param("Build and deploy", "Compile assets", False, id="nontest-job-and-nontest-step"),- ],-)-def test_is_test_job_failure(job_name: str, failed_step: str, expected: bool) -> None:- assert is_test_job_failure(make_job(job_name, failed_step=failed_step)) is expected---def test_build_decision_events_one_event_per_decision() -> None:- decisions = (- classify_job(make_job("Django tests - Temporal (1/1)")),- classify_job(make_job("Build and deploy", failed_step="Compile assets")),- classify_job(make_job("Validate OpenAPI types", failed_step="Check and update OpenAPI types")),- )-- events = build_decision_events("PostHog/posthog", make_workflow_run(), decisions)-- assert [event["event"] for event in events] == [DECISION_EVENT, DECISION_EVENT, DECISION_EVENT]- assert all(event["distinct_id"] == "PostHog/posthog" for event in events)- first = events[0]["properties"]- assert first["action"] == "observe"- assert first["workflow_name"] == "Backend CI"- assert first["job_name"] == "Django tests - Temporal (1/1)"- assert first["$groups"] == {"workflow_run": "999"}- assert first["classified_via"] == "job_name"- assert first["failure_class"] is None- assert first["pr_number"] == 12345- assert first["job_conclusion"] == "failure"- assert first["failed_steps"] == ["Run Core tests"]- assert first["$insert_id"] == "decision:999:1:123"- assert events[1]["properties"]["action"] == "skip non-test"- assert events[1]["properties"]["classified_via"] is None- assert events[2]["properties"]["action"] == "skip deterministic"- assert events[2]["properties"]["failure_class"] == "openapi"--- ("pull_requests", "expected"),- [- pytest.param([{"number": 65581}], 65581, id="same-repo-pr"),- pytest.param([], None, id="fork-pr-empty-list"),- pytest.param(None, None, id="field-absent"),- ],-)-def test_workflow_run_pr_number(pull_requests: list[dict[str, object]] | None, expected: int | None) -> None:- raw: dict[str, object] = {"id": 1, "workflow_id": 2, "name": "Backend CI"}- if pull_requests is not None:- raw["pull_requests"] = pull_requests-- assert workflow_run_from_object(raw).pr_number == expected--- ("conclusion", "expected"),- [- pytest.param("success", "cleared", id="success-cleared"),- pytest.param("failure", "still_failing", id="failure-still-failing"),- pytest.param("timed_out", "still_failing", id="timed-out-still-failing"),- pytest.param(None, "unknown", id="missing-unknown"),- pytest.param("cancelled", "unknown", id="cancelled-unknown"),- ],-)-def test_rerun_outcome_label(conclusion: str | None, expected: str) -> None:- assert rerun_outcome_label(conclusion) == expected---def test_report_rerun_outcomes_empty_on_first_attempt() -> None:- assert report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=1)) == []--- ("current_conclusion", "expected_outcome"),- [- pytest.param("success", "cleared", id="rerun-cleared"),- pytest.param("failure", "still_failing", id="rerun-still-failing"),- pytest.param("timed_out", "still_failing", id="rerun-timed-out"),- ],-)-def test_report_rerun_outcomes_attributes_prior_test_failure(- monkeypatch: pytest.MonkeyPatch, current_conclusion: str, expected_outcome: str-) -> None:- job_name = "Django tests - Temporal (1/1)"-- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:- if attempt == 1:- return (make_job(job_name, conclusion="failure", started_at="2026-06-05T21:00:00Z"),)- # Advanced started_at: this attempt genuinely re-executed the job.- return (make_job(job_name, conclusion=current_conclusion, started_at="2026-06-05T22:00:00Z"),)-- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)-- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))-- assert len(events) == 1- assert events[0]["event"] == OUTCOME_EVENT- props = events[0]["properties"]- assert props["outcome"] == expected_outcome- assert props["job_name"] == job_name- assert props["prior_conclusion"] == "failure"- assert props["prior_attempt"] == 1- assert props["attempt"] == 2- assert props["$insert_id"] == f"outcome:999:2:{job_name}"---def test_report_rerun_outcomes_not_rerun_when_started_at_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:- # The regression this fix targets: a failed job carried over (never re-executed) keeps its prior- # started_at and conclusion. It must read as `not_rerun`, never `still_failing`.- job_name = "Django tests - Temporal (1/1)"-- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:- return (make_job(job_name, conclusion="failure", started_at="2026-06-05T21:00:00Z"),)-- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)-- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))-- assert len(events) == 1- assert events[0]["properties"]["outcome"] == "not_rerun"---def test_report_rerun_outcomes_unknown_when_job_absent(monkeypatch: pytest.MonkeyPatch) -> None:- job_name = "Django tests - Temporal (1/1)"-- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:- if attempt == 1:- return (make_job(job_name, conclusion="failure"),)- return () # job missing from the re-run attempt-- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)-- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))-- assert len(events) == 1- assert events[0]["properties"]["outcome"] == "unknown"---def test_report_rerun_outcomes_unknown_when_current_name_ambiguous(monkeypatch: pytest.MonkeyPatch) -> None:- job_name = "Django tests - Temporal (1/1)"-- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:- if attempt == 1:- return (make_job(job_name, conclusion="failure"),)- # Two current jobs share a name: ambiguous, so it can't be paired and must read as unknown.- return (- make_job(job_name, conclusion="success", started_at="2026-06-05T22:00:00Z"),- make_job(job_name, conclusion="failure", started_at="2026-06-05T22:00:00Z"),- )-- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)-- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))-- assert len(events) == 1- assert events[0]["properties"]["outcome"] == "unknown"--- ("head_branch", "event", "expected_is_master"),- [- pytest.param("master", "push", True, id="master-is-master"),- pytest.param("main", "push", True, id="main-is-master"),- pytest.param("some-feature", "pull_request", False, id="pr-branch-not-master"),- ],-)-def test_build_decision_events_is_master(head_branch: str, event: str, expected_is_master: bool) -> None:- decisions = (classify_job(make_job("Django tests - Temporal (1/1)")),)-- events = build_decision_events(- "PostHog/posthog", make_workflow_run(head_branch=head_branch, event=event), decisions- )-- props = events[0]["properties"]- assert props["is_master"] is expected_is_master- assert props["trigger_event"] == event---def test_report_rerun_outcomes_ignores_non_test_jobs(monkeypatch: pytest.MonkeyPatch) -> None:- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:- return (make_job("Build and deploy", conclusion="failure", failed_step="Compile assets"),)-- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)-- assert report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2)) == []---def _raise_if_called(*args: object, **kwargs: object) -> None:- raise AssertionError("telemetry should not POST")---def test_capture_events_noop_without_api_key(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(ci_flake_overseer.urllib.request, "urlopen", _raise_if_called)-- capture_events("", "https://us.i.posthog.com", [{"event": DECISION_EVENT}])---def test_capture_events_noop_without_events(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(ci_flake_overseer.urllib.request, "urlopen", _raise_if_called)-- capture_events("phc_test", "https://us.i.posthog.com", [])---def test_capture_events_posts_batch(monkeypatch: pytest.MonkeyPatch) -> None:- captured: dict[str, object] = {}-- class FakeResponse:- def __enter__(self) -> FakeResponse:- return self-- def __exit__(self, *args: object) -> bool:- return False-- def fake_urlopen(request: object, timeout: object = None) -> FakeResponse:- captured["url"] = request.full_url # type: ignore[attr-defined]- captured["body"] = json.loads(request.data.decode()) # type: ignore[attr-defined]- return FakeResponse()-- monkeypatch.setattr(ci_flake_overseer.urllib.request, "urlopen", fake_urlopen)- events = [{"event": DECISION_EVENT, "distinct_id": "PostHog/posthog", "properties": {}}]-- capture_events("phc_test", "https://us.i.posthog.com/", events)-- assert captured["url"] == "https://us.i.posthog.com/batch/"- assert captured["body"] == {"api_key": "phc_test", "batch": events}---def test_decision_is_minimal_dataclass() -> None:- decision = Decision(action="observe", reason="x", job=make_job("Django tests - Temporal (1/1)"))- assert decision.action == "observe"diff --git a/.github/scripts/test_ci_migration_selection.py b/.github/scripts/test_ci_migration_selection.pyindex 5b29c0116703..3105ef0847cf 100644--- a/.github/scripts/test_ci_migration_selection.py+++ b/.github/scripts/test_ci_migration_selection.py@@ -38,13 +38,16 @@ def commit_file(repo: Path, name: str, content: str) -> str:def commit_on_master(repo: Path, master: str, name: str, content: str) -> None:git(repo, "checkout", "-q", "master")commit_file(repo, name, content)- git(repo, "update-ref", f"refs/remotes/{master}", "HEAD")+ git(repo, "update-ref", f"refs/remotes/{MASTER}", "HEAD")git(repo, "checkout", "-q", "topic")+MASTER = "upstream/master"++@functools.cache-def list_step(engine: str) -> str:- workflow = yaml.safe_load((ROOT / engine / "workflows/ci-backend.yml").read_text())+def list_step() -> str:+ workflow = yaml.safe_load((ROOT / ".depot/workflows/ci-backend.yml").read_text())return next(step["run"]for job in workflow["jobs"].values()@@ -53,7 +56,6 @@ def list_step(engine: str) -> str:)[email protected]("engine,master", [(".github", "origin/master"), (".depot", "upstream/master")])@pytest.mark.parametrize("case,expected",[@@ -69,8 +71,6 @@ def list_step(engine: str) -> str:],)def test_lists_only_this_prs_migrations(- engine: str,- master: str,case: str,expected: tuple[list[str], list[str]] | None,tmp_path: Path,@@ -83,7 +83,7 @@ def test_lists_only_this_prs_migrations(git(repo, "config", "commit.gpgsign", "false")base = commit_file(repo, "0001_base.py", "base\n")commit_file(repo, "0002_master.py", "master\n")- git(repo, "update-ref", f"refs/remotes/{master}", "HEAD")+ git(repo, "update-ref", f"refs/remotes/{MASTER}", "HEAD")git(repo, "checkout", "-qb", "topic")if case == "based on master":@@ -92,10 +92,10 @@ def test_lists_only_this_prs_migrations(elif case == "stale base":commit_file(repo, "0003_own.py", "own\n")elif case == "master adds a similar migration later":- commit_on_master(repo, master, "0003_master.py", MIGRATION_TEMPLATE.format(table="master_table"))+ commit_on_master(repo, MASTER, "0003_master.py", MIGRATION_TEMPLATE.format(table="master_table"))commit_file(repo, "0003_own.py", MIGRATION_TEMPLATE.format(table="own_table"))elif case == "master edits a migration the branch carries":- commit_on_master(repo, master, "0002_master.py", "master fixed\n")+ commit_on_master(repo, MASTER, "0002_master.py", "master fixed\n")elif case == "edits an existing migration":commit_file(repo, "0001_base.py", "modified\n")elif case == "base missing from the clone":@@ -107,16 +107,15 @@ def test_lists_only_this_prs_migrations(["git", "clone", "-q", "--depth=1", "--no-single-branch", "-b", "topic", source.as_uri(), str(repo)],check=True,)- if engine == ".depot":- git(repo, "update-ref", f"refs/remotes/{master}", "refs/remotes/origin/master")- git(repo, "remote", "remove", "origin")+ git(repo, "update-ref", f"refs/remotes/{MASTER}", "refs/remotes/origin/master")+ git(repo, "remote", "remove", "origin")elif case == "unknown base":base = "0" * 40elif case == "no master ref":- git(repo, "update-ref", "-d", f"refs/remotes/{master}")+ git(repo, "update-ref", "-d", f"refs/remotes/{MASTER}")result = subprocess.run(- ["bash", "-e", "-c", list_step(engine)],+ ["bash", "-e", "-c", list_step()],cwd=repo,env={**os.environ,diff --git a/.github/scripts/test_depot_backend_runs.py b/.github/scripts/test_depot_backend_runs.pyindex d3acdbdde02a..182344dbf02a 100644--- a/.github/scripts/test_depot_backend_runs.py+++ b/.github/scripts/test_depot_backend_runs.py@@ -7,7 +7,6 @@ from typing import Anyimport pytest-import ci_backend_relayimport depot_backend_runs as scriptMASTER_SHA = "a" * 40@@ -50,11 +49,6 @@ def test_gate_run_reports_the_gate_verdict(assert run["head_sha"] == "abc"-def test_names_the_workflow_and_gate_job_the_relay_reads() -> None:- assert script.WORKFLOW_NAME == ci_backend_relay.DEPOT_WORKFLOW- assert script.GATE_JOB_KEY == ci_backend_relay.GATE_JOB_KEY--@pytest.mark.parametrize("trigger,sha,kept",[("push", MASTER_SHA, 1), ("push", BRANCH_SHA, 0), ("schedule", BRANCH_SHA, 1)],diff --git a/.github/scripts/trunk-cross-lane.test.js b/.github/scripts/trunk-cross-lane.test.jsindex 7ecb5f780215..a1766ed046de 100644--- a/.github/scripts/trunk-cross-lane.test.js+++ b/.github/scripts/trunk-cross-lane.test.js@@ -38,7 +38,7 @@ for (const [name, files, mixed] of [['prose beside Node', ['README.md', 'nodejs/src/ingestion/ingestion-consumer.ts'], false],['a Python workflow beside a Rust workflow',- ['.github/workflows/ci-backend.yml', '.github/workflows/ci-rust.yml'],+ ['.depot/workflows/ci-backend.yml', '.github/workflows/ci-rust.yml'],false,],]) {diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.jsindex 8010f8b915ae..be7435045b51 100644--- a/.github/scripts/trunk-impacted-targets.js+++ b/.github/scripts/trunk-impacted-targets.js@@ -253,9 +253,10 @@ const TRIPWIRE_RULES = [['.github/workflows/ci-nodejs.yml', NODE],['.github/workflows/ci-nodejs-container.yml', NODE],['.github/workflows/ci-mcp.yml', JAVASCRIPT],- ['.github/workflows/ci-backend.yml', PYTHON],+ ['.depot/workflows/ci-backend.yml', PYTHON],['.github/workflows/ci-backend-update-test-timing.yml', PYTHON],- ['.github/workflows/ci-backend-shadow-drift.yml', PYTHON],+ ['.github/workflows/ci-backend-schema-mirror.yml', PYTHON],+ ['.github/workflows/ci-depot-action-mirrors.yml', PYTHON],['.github/workflows/ci-dagster.yml', PYTHON],['.github/workflows/ci-rust.yml', RUST],['.github/workflows/ci-rust-flags-integration.yml', RUST],@@ -411,9 +412,6 @@ const TRIPWIRE_RULES = [['.github/scripts/eng-analytics-weekly-digest.mjs', REPO_AUTOMATION],['.github/scripts/ci-alerts-devex.js', REPO_AUTOMATION],['.github/scripts/ci-alerts-devex.test.js', REPO_AUTOMATION],- ['.github/scripts/ci_flake_overseer.py', REPO_AUTOMATION],- ['.github/scripts/test_ci_flake_overseer.py', REPO_AUTOMATION],- ['.github/scripts/compare-ci-runners.py', REPO_AUTOMATION],['.github/scripts/report_workflow_run_traces.py', REPO_AUTOMATION],['.github/scripts/test_report_workflow_run_traces.py', REPO_AUTOMATION],['.github/auto-assign-labels.json', REPO_AUTOMATION],@@ -589,6 +587,7 @@ const TRIPWIRE_RULES = [['rust/.sqlx/**', RUST],['hogli.yaml', UNIVERSAL],['.github/**', UNIVERSAL],+ ['.depot/workflows/**', UNIVERSAL],['docker-compose*.yml', UNIVERSAL],// Single-purpose images ahead of the fallback: each is read by exactly one// workflow or suite, whose rule above already carries the radius.@@ -931,16 +930,13 @@ function globToRegExp(glob) {const TRIPWIRE_MATCHERS = TRIPWIRE_RULES.map(([glob, domain]) => [globToRegExp(glob), domain])-// .depot/ holds Depot-runner shadows of the workflows and composite actions in-// .github/, kept apples-to-apples with their canonicals by the shadow-drift-// check, and their statuses are non-blocking. A shadow can only affect what-// its canonical affects, so it resolves through the canonical's rules: the-// ci-backend shadow takes the python lanes, and a shadow of something unplaced-// still lands on the .github/** blanket.-const DEPOT_MIRROR_PREFIX = '.depot/'+// .depot/actions/ holds Depot CI's mirrors of the composite actions in .github/actions/,+// kept in sync by the action-mirrors check, so a mirror resolves through its canonical+// action's rules. Depot CI's own workflows under .depot/workflows/ carry rules of their own.+const DEPOT_MIRROR_PREFIX = '.depot/actions/'function canonicalPath(file) {- return file.startsWith(DEPOT_MIRROR_PREFIX) ? `.github/${file.slice(DEPOT_MIRROR_PREFIX.length)}` : file+ return file.startsWith(DEPOT_MIRROR_PREFIX) ? `.github/actions/${file.slice(DEPOT_MIRROR_PREFIX.length)}` : file}// The domain of the first rule matching the file, or null when no rule claimsdiff --git a/.github/scripts/trunk-impacted-targets.test.js b/.github/scripts/trunk-impacted-targets.test.jsindex b35ec963b4c7..a14e884d908a 100644--- a/.github/scripts/trunk-impacted-targets.test.js+++ b/.github/scripts/trunk-impacted-targets.test.js@@ -135,7 +135,7 @@ test('a universal tripwire claims every known target', () => {// included.'docker-compose.base.yml',// Trees that steer what every suite runs or what it runs against: the- // Depot shadow of an action every workflow uses, the toolchain, the+ // Depot mirror of an action every workflow uses, the toolchain, the// service configs the stack mounts, and the markdownlint config every// tree's prose obeys.'.depot/actions/pnpm-install/action.yml',@@ -525,7 +525,7 @@ test('the quarantine list claims the fullstack lanes and no rust crate', () => {// paths-filter decides which jobs run inside a single run's own diff, so no// run's outcome depends on it jointly with another queue entry. Its own CI and-// its .depot shadow share the lane.+// its .depot mirror share the lane.test('the paths-filter action and its CI share the ci-tooling lane', () => {for (const file of ['.github/actions/paths-filter/src/filter.ts',@@ -601,7 +601,7 @@ test('a single-language workflow claims that language rather than everything', ()}for (const file of [- '.github/workflows/ci-backend.yml',+ '.depot/workflows/ci-backend.yml','.github/workflows/ci-python.yml','.github/workflows/ci-clickhouse-multinode-migrations.yml','.github/workflows/ci-clickhouse-util-udfs.yml',@@ -781,20 +781,9 @@ test('the schema codegen scripts claim the product-surface lanes', () => {)})-// A Depot shadow is kept apples-to-apples with its canonical by the-// shadow-drift check and posts non-blocking statuses, so it can only affect-// what the canonical affects. The narrowing direction is the guard: a shadow-// of something no rule has placed must stay on the .github blanket rather-// than fall through unclaimed.-test('a depot shadow resolves through its canonical workflow rules', () => {- assert.deepEqual(- computeTargets(['.depot/workflows/ci-backend.yml'], CONTEXT),- computeTargets(['.github/workflows/ci-backend.yml'], CONTEXT)- )- assert.deepEqual(- computeTargets(['.depot/workflows/ci-backend-update-test-timing.yml'], CONTEXT),- computeTargets(['mypy.ini'], CONTEXT)- )+// A Depot action mirror resolves through its canonical action, and a Depot workflow no rule+// has placed stays on its own blanket rather than falling through unclaimed.+test('depot mirrors and unplaced depot workflows resolve to everything', () => {assert.deepEqual(computeTargets(['.depot/actions/pnpm-install/action.yml'], CONTEXT), EVERYTHING)assert.deepEqual(computeTargets(['.depot/workflows/some-new-shadow.yml'], CONTEXT), EVERYTHING)// Shadow markdown is prose like any other.diff --git a/.github/scripts/turbo-discover.test.js b/.github/scripts/turbo-discover.test.jsindex 62faf1f68135..02a2278de10f 100644--- a/.github/scripts/turbo-discover.test.js+++ b/.github/scripts/turbo-discover.test.js@@ -3,8 +3,7 @@// Pins turbo-discover's DJANGO_SEGMENTS table to the Django pytest invocations// in ci-backend.yml. The table sizes the shards each segment gets, so a segment// listing paths the workflow no longer runs (or missing ones it does) budgets-// wall time for a run that never happens. Both workflow copies are read: the-// Depot mirror runs the same matrix and drifts on its own.+// wall time for a run that never happens.//// Reads the workflow rather than a fixture on purpose — the workflow is the one// side that can drift, and there is nothing else to compare the table against.@@ -19,7 +18,7 @@ const path = require('node:path')const { DJANGO_SEGMENTS, getIsolatedProducts, loadJsonTargets } = require('./turbo-discover')const REPO_ROOT = path.join(__dirname, '..', '..')-const WORKFLOWS = ['.github/workflows/ci-backend.yml', '.depot/workflows/ci-backend.yml']+const WORKFLOW = '.depot/workflows/ci-backend.yml'// pytest takes `./posthog/queries/` and `posthog` for the same directory, and// --ignore drops the trailing slash. DJANGO_SEGMENTS stores prefixes, so every@@ -68,21 +67,19 @@ function parseTemporalTargets(text) {return invocation[1].split(/\s+/)}-for (const workflow of WORKFLOWS) {- const text = fs.readFileSync(path.join(REPO_ROOT, workflow), 'utf8')- const core = parseCoreStep(text)- const expected = {- Core: { include: core.core, exclude: core.ignores },- CorePOE: { include: core.poe, exclude: [...core.ignores, ...core.poeIgnores] },- Temporal: { include: parseTemporalTargets(text), exclude: [] },- }+const text = fs.readFileSync(path.join(REPO_ROOT, WORKFLOW), 'utf8')+const core = parseCoreStep(text)+const expected = {+ Core: { include: core.core, exclude: core.ignores },+ CorePOE: { include: core.poe, exclude: [...core.ignores, ...core.poeIgnores] },+ Temporal: { include: parseTemporalTargets(text), exclude: [] },+}- for (const [segment, paths] of Object.entries(expected)) {- test(`${segment} segment matches the pytest targets in ${workflow}`, () => {- assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].include), sorted(paths.include.map(toPrefix)))- assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].exclude), sorted(paths.exclude.map(toPrefix)))- })- }+for (const [segment, paths] of Object.entries(expected)) {+ test(`${segment} segment matches the pytest targets in ${WORKFLOW}`, () => {+ assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].include), sorted(paths.include.map(toPrefix)))+ assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].exclude), sorted(paths.exclude.map(toPrefix)))+ })}// The backend test selector is the third copy of the partition: it routes eachdiff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.ymldeleted file mode 100644index 2439c30e0a98..000000000000--- a/.github/workflows/ci-backend.yml+++ /dev/null@@ -1,4661 +0,0 @@-# This workflow runs all of our backend django tests.-#-# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching-# .github/workflows/ci-backend-update-test-timing.yml action-name: Backend CI-on:- # No push or schedule trigger: the master push and the hourly full-matrix master run are on- # Depot CI, from .depot/workflows/ci-backend.yml.- workflow_dispatch:- inputs:- clickhouseServerVersion:- description: ClickHouse server version. Leave blank for default- type: string- pull_request:- # PRs run the snob-selected Django subset (one shard per active segment)- # for fast feedback, draft or ready. When selection can't be trusted, a- # ready PR falls back to the full matrices and a draft skips them instead,- # deferring to its ready run. turbo-tests (product tests) still skip drafts.- # On a legacy diff the product matrix is narrowed the same way: the products the- # selector reached plus the ones Turbo saw change (see turbo-discover).- # The merge queue's trunk-merge/** run and the hourly scheduled master run- # always run the full matrices; the queue run is what actually gates master.- # Master pushes run only the per-commit checks (migrations + schema cache,- # OpenAPI types, repo checks). To force the full matrices on a PR, add the- # `run-ci-backend` label. Cheap checks still run on drafts.- # The `no-ci` label does the opposite: it silences this workflow on a draft- # entirely (prototypes/spikes).- # No labeled/unlabeled triggers: GitHub cannot filter a label trigger by name,- # so every unrelated label re-ran these matrices against a commit CI had- # already covered. A label here takes effect from the next push, or from- # ready_for_review.- # The events_json leg runs wherever the legacy tests run, and on every third- # hourly master run: the paths in .github/new-events-schema-targets.txt rerun against the- # native-JSON events table, all of them on full runs and the reached ones on- # narrowed runs. A PR whose product-only change forced the full run reruns only- # that product's paths, because the merge queue runs the whole list before merge.- # The `test-new-events-schema` label opts a PR into the doubled- # matrices that run the whole suite in both events schemas, in place of that leg.- # Apply it before a push or ready_for_review because label events do not start- # this workflow. Manual dispatches also run both modes.- types: [opened, synchronize, reopened, ready_for_review]--concurrency:- # PRs: one active run per branch, cancel stale. Push: per-SHA so master- # pushes never cancel each other (check-migrations always completes).- group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.head_ref || github.ref }}- cancel-in-progress: ${{ github.event_name == 'pull_request' }}--env:- # Schema cache epoch. Bump to abandon every shared schema cache entry at once- # (key is posthog-schema-mig-<epoch>-<hash>). Single-sourced so it can't drift- # between the merge-base (PR) and HEAD (push) key computations below.- # ci-e2e-playwright.yml, ci-dagster.yml, ci-mcp.yml and ci-rust-flags-integration.yml restore by the same key; keep their copies in sync when bumping.- SCHEMA_CACHE_EPOCH: v2- # Hypothesis constants cache epoch. Bump to abandon every shared entry at once- # (key is posthog-hypothesis-constants-<version>-<epoch>-<isoyear>-<isoweek>); used by- # the Django test shards below.- HYPOTHESIS_CONSTANTS_EPOCH: v1- SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only- DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'- REDIS_URL: 'redis://localhost'- CLICKHOUSE_HOST: 'localhost'- CLICKHOUSE_SECURE: 'False'- CLICKHOUSE_VERIFY: 'False'- # Restricted `autoresearch` user defined in docker/clickhouse/users-dev.xml,- # pointed at the CI test DB (posthog_test, not `default` as in regular dev).- CLICKHOUSE_TEST_CLUSTER_HOST: 'localhost'- CLICKHOUSE_TEST_CLUSTER_DATABASE: 'posthog_test'- CLICKHOUSE_TEST_CLUSTER_USER: 'autoresearch'- CLICKHOUSE_TEST_CLUSTER_PASSWORD: 'autoresearchpass'- CLICKHOUSE_TEST_CLUSTER_SECURE: 'False'- CLICKHOUSE_TEST_CLUSTER_VERIFY: 'False'- TEST: 1- # The root conftest ends a green pytest session with os._exit once every report is written.- # The hourly run on Depot CI turns this off, because it also hides a crash during interpreter teardown.- POSTHOG_PYTEST_HARD_EXIT: '1'- CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}- CLICKHOUSE_COMPAT_PYTEST_TARGETS: 'posthog/clickhouse ee/clickhouse'- OBJECT_STORAGE_ENABLED: 'True'- OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'- OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'- OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'- UV_HTTP_TIMEOUT: 120- # A 4 GiB thread stack cache avoids the uv exit segfault (code 139) on glibc before 2.43.- # Remove when uv ships the fix: https://github.com/astral-sh/uv/issues/21428- GLIBC_TUNABLES: glibc.pthread.stack_cache_size=0x100000000- # tests would intermittently fail in GH actions- # with exit code 134 _after passing_ all tests- # this appears to fix it- # absolute wild tbh https://stackoverflow.com/a/75503402- DISPLAY: ':99.0'- # this is a fake key so this workflow can run for external contributors as they do not have access to secrets (that we don't need here)- OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }}- RUNS_ON_INTERNAL_PR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}- SANDBOX_JWT_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }}--permissions:- contents: read- actions: read- pull-requests: write--jobs:- # Learning mode: nothing reads these outputs, so a recommendation cannot change what runs.- # Fork and Dependabot runs get no secret; the queue always runs everything anyway.- # The step fails open, because this telemetry must not turn a passing workflow red.- dynamic-ci-filter:- name: Trunk Dynamic CI (Learning Mode)- if: >-- github.event_name == 'pull_request' &&- github.repository == 'PostHog/posthog' &&- github.event.pull_request.head.repo.full_name == github.repository &&- github.actor != 'dependabot[bot]' &&- !startsWith(github.head_ref, 'trunk-merge/') &&- (github.event.pull_request.draft != true ||- !contains(github.event.pull_request.labels.*.name, 'no-ci'))- runs-on: ubuntu-24.04- timeout-minutes: 5- permissions: {}- steps:- - name: Ask Trunk which jobs this diff needs- continue-on-error: true- uses: trunk-io/dynamic-ci@7e3af9331e8ebdfe0c71ba7d0ff6b7424dde8c57 # v1- with:- token: ${{ secrets.TRUNK_API_TOKEN }}-- # Job to decide if we should run backend ci- # See .github/actions/paths-filter/README.md for filter semantics- changes:- runs-on: depot-ubuntu-24.04- timeout-minutes: 5- # Depot CI runs every pull request event from .depot/workflows/ci-backend.yml, and- # its gate posts the required check. Every job that needs this one skips on a pull- # request, and the gate treats skipped as success.- if: github.event_name != 'pull_request'- name: Determine need to run backend and migration checks- # Set job outputs to values from filter step- outputs:- # run-ci-backend is documented as forcing the full matrices, but every- # gate reads this output, so the label cannot reach them while the paths- # filter says false. Nothing under .github/ is in the backend filter, so- # a sharding or timing script change was untestable on its own PR.- backend: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && 'true' || (steps.filter.outputs.backend || 'true') }}- backend_files: ${{ steps.filter.outputs.backend_files }}- migrations: ${{ steps.filter.outputs.migrations || 'true' }}- # Added and modified files only. A squash retirement deletes thousands of- # migrations, and a step env string over 128 KiB fails the spawn (E2BIG).- migrations_review_files: ${{ steps.filter.outputs.migrations_review_files }}- migrations_any: ${{ steps.filter.outputs.migrations_any || 'true' }}- # What the replay gate in check-migrations keys off. A ClickHouse or- # async-migration change does not pay for a replay of a history the Django- # planner never sees.- migration_replay: ${{ steps.filter.outputs.migration_replay || 'true' }}- persons_sql: ${{ steps.filter.outputs.persons_sql || 'true' }}- tasks_temporal: ${{ steps.filter.outputs.tasks_temporal || 'true' }}- openapi_types: ${{ steps.filter.outputs.openapi_types || 'true' }}- # The label has to reach discovery as well: `backend` alone only starts the- # discover job, which then sees no legacy change and no affected product and- # emits an empty matrix. A forced legacy change makes it test everything.- legacy: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && 'true' || (steps.filter.outputs.legacy || 'true') }}- schema: ${{ steps.filter.outputs.schema || 'true' }}- product_yamls: ${{ steps.filter.outputs.product_yamls || 'false' }}- repo_invariants: ${{ steps.filter.outputs.repo_invariants || 'false' }}- sdk_manifests: ${{ steps.filter.outputs.sdk_manifests || 'true' }}- product_yamls_files: ${{ steps.filter.outputs.product_yamls_files }}- timing_scripts: ${{ steps.filter.outputs.timing_scripts }}- steps:- # For pull requests it's not necessary to checkout the code, but we- # also want this to run on master so we need to checkout- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- clean: false- sparse-checkout: |- .github/actions/paths-filter- sparse-checkout-cone-mode: false- - name: Clean up data directories with container permissions- run: |- # Use docker to clean up files created by containers- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true- continue-on-error: true-- - uses: ./.github/actions/paths-filter- # Skipped on master push: every output then defaults to true.- if: github.event_name != 'push'- id: filter- with:- token: ${{ github.token }}- list-files: 'escape'- filters: |- backend:- # Avoid running backend tests for irrelevant changes- # NOTE: we are at risk of missing a dependency here. We could make- # the dependencies more clear if we separated the backend/frontend- # code completely- - 'ee/**/*'- - '!ee/frontend/**' # frontend code under ee/ must not trigger backend- - 'common/__init__.py'- - 'common/hogql_parser/**'- - 'rust/hogql/parser/**'- - 'common/hogvm/**'- - 'common/ingestion/**'- - 'common/migration_utils/**'- - 'common/plugin_transpiler/**'- # The personhog gRPC stubs that posthog/personhog_client imports- - 'packages/personhog-proto/**'- - 'posthog/**/*'- - 'products/**/backend/**/*'- # Python outside backend/ — products like logs and- # posthog_ai point `backend:test` at extra roots- # (skills/, scripts/). Catch any .py change so the- # backend job triggers when only those files move.- - 'products/**/*.py'- # products/desktop is the imported desktop app; it has its own- # desktop-* CI and must not drag this suite onto its PRs- - '!products/desktop/**'- # Each product's package.json declares the per-product- # backend:test script that turbo-discover reads to- # compose the Product matrix. Without this entry,- # widening or fixing the script doesn't trigger- # tests it controls.- - 'products/*/package.json'- # Tests read these MCP manifests from disk. Each one is a turbo input of- # its reader (tasks directly, posthog_ai through feature_flags'- # backend:mcp-surface task). Add a manifest here only with that wiring.- - products/feature_flags/mcp/tools.yaml- - products/tasks/mcp/tools.yaml- - 'products/canvas/packages/canvas_builder/**'- - bin/build-schema-latest-versions.py- - bin/check_uv_python_compatibility.py- - bin/find_python_dependencies.py- - pyproject.toml- - uv.lock- - requirements.txt- - requirements-dev.txt- - mypy.ini- - pytest.ini- - conftest.py # Root conftest loads for every pytest run- - .test_quarantine.json # Un-quarantining must re-run the affected tests- # tach.toml's exclude list shapes the import map turbo-discover- # reads for the dependent cascade, and `tach check` validates- # the declarations — a graph-only PR must still dispatch both.- - tach.toml- # generates posthog/schema.py- - frontend/src/queries/schema.json- # Operator parity check- - rust/feature-flags/src/properties/property_models.rs- - frontend/src/products.json # Loaded at runtime by posthog/products.py- - 'products/*/manifest.tsx' # Generates products.json- # hogli setup commands this workflow runs: db:restore-test-db- # and product:bootstrap/lint. Scoped to the modules those- # commands actually import — framework, manifest, boot- # modules, db_schema, product — so unrelated hogli commands- # (doctor, devbox, ...) don't trigger this suite.- - 'tools/hogli/**'- - hogli.yaml- - 'tools/hogli-commands/hogli_commands/db_schema.py'- - 'tools/hogli-commands/hogli_commands/prechecks.py'- - 'tools/hogli-commands/hogli_commands/telemetry_props.py'- - 'tools/hogli-commands/hogli_commands/hint_hook.py'- - 'tools/hogli-commands/hogli_commands/hints.py'- - 'tools/hogli-commands/hogli_commands/product/**'- - 'tools/hogli-commands/hogli_commands/product_structure.yaml'- # owners_yaml is a runtime dependency of product code (stamphog reads- # owners.yaml through it), so a resolver change has to re-run its- # dependents; turbo-discover finds the products that import it.- # Deliberately not in `legacy`, because a leaf-package change should- # run its dependents rather than the whole Django suite.- - 'packages/owners-yaml/**'- # Transitional: branches that predate the move still carry the- # resolver at tools/owners.- - 'tools/owners/**'- # product:lint --all checks these generated files against the tree,- # so a baseline-only edit must run it too. The crossings baseline is- # also what product:lint reads to let a wiring location leave a- # product's turbo.json inputs, so both edits must run the checks.- - products/isolation_baseline.txt- - products/model_crossing_uses_baseline.txt- - 'products/*/turbo.json'- # Make sure we run if someone is explicitly changing the workflow- - .github/workflows/ci-backend.yml- - .github/new-events-schema-targets.txt- - .github/actions/setup-uv/**- - .github/clickhouse-versions.json- # We use docker compose for tests, make sure we rerun on- # changes to docker-compose.dev.yml e.g. dependency- # version changes- - docker-compose.dev.yml- - docker-compose.profiles.yml- - docker-compose.base.yml- - bin/wait-for-docker- - bin/ci-wait-for-docker- - frontend/public/email/*- - 'docker/clickhouse/**'- # Markdown-only changes (e.g. a README under posthog/) match the- # broad globs above but exercise no backend code — exclude them so- # a docs-only PR doesn't run the whole backend suite.- # These excludes are load-bearing: an over-broad exclude silently- # skips the suite rather than over-running it, so only exclude- # patterns that truly never affect backend. They also drop matched- # files from `backend_files`.- - '!**/*.md'- - '!**/*.mdx'- legacy:- # Non-product backend code — when only products/ change,- # turbo-discover uses Turbo query affectedness to detect changed- # products and decide whether Django runs.- # Everything from backend: EXCEPT products/**/backend/**/*- - 'ee/**/*'- - '!ee/frontend/**' # frontend code under ee/ must not trigger backend- - 'common/__init__.py'- - 'common/hogql_parser/**'- - 'rust/hogql/parser/**'- - 'common/hogvm/**'- - 'common/ingestion/**'- - 'common/migration_utils/**'- - 'common/plugin_transpiler/**'- # The personhog gRPC stubs that posthog/personhog_client imports- - 'packages/personhog-proto/**'- - 'posthog/**/*'- - bin/build-schema-latest-versions.py- - bin/check_uv_python_compatibility.py- - bin/find_python_dependencies.py- - pyproject.toml- - uv.lock- - requirements.txt- - requirements-dev.txt- - mypy.ini- - pytest.ini- - conftest.py # Root conftest loads for every pytest run- - .test_quarantine.json- - frontend/src/products.json- - 'products/*/manifest.tsx'- - rust/feature-flags/src/properties/property_models.rs- - .github/workflows/ci-backend.yml- - .github/new-events-schema-targets.txt- - .github/actions/setup-uv/**- - .github/clickhouse-versions.json- - docker-compose.dev.yml- - docker-compose.profiles.yml- - docker-compose.base.yml- - bin/wait-for-docker- - bin/ci-wait-for-docker- - frontend/public/email/*- - 'docker/clickhouse/**'- # Same rationale as the backend filter's excludes above: docs-only- # changes match the broad globs but exercise no legacy backend code,- # so they shouldn't trigger the Django/turbo-discover path either.- - '!**/*.md'- - '!**/*.mdx'- # Generated from frontend/src/queries/schema.json (see the `schema`- # filter below). Excluding them here routes schema-only PRs into- # turbo-discover's schema-impact narrowing instead of the- # all-products legacy path. This cannot skip Django: the- # schemaChanged branch hard-sets run_legacy=true. It also cannot- # hide a hand-edit to the generated files — ci-python.yml- # regenerates schema.py and fails on any diff, and both files stay- # in the `schema` (and `backend`) filters so turbo-discover still- # takes the schema path for them.- - '!posthog/schema.py'- - '!posthog/schema_enums.py'- schema:- # Tracked separately from `legacy` so turbo-discover can- # diff schema.json and narrow the product matrix to- # products that import the changed types.- - frontend/src/queries/schema.json- # The generated outputs are excluded from `legacy` above; listing- # them here keeps run_legacy=true (full Django) for any schema- # change, including a stray hand-edit to a generated file.- - posthog/schema.py- - posthog/schema_enums.py- migrations:- - 'docker/clickhouse/**'- - 'posthog/migrations/*.py'- - 'products/*/backend/migrations/*.py'- - 'products/*/migrations/*.py' # Legacy structure- migrations_review:- - added|modified: 'posthog/migrations/*.py'- - added|modified: 'products/*/backend/migrations/*.py'- - added|modified: 'products/*/migrations/*.py'- # Any migration file in any app (broader than `migrations` above,- # which omits ee/ and posthog sub-package dirs). Gates the- # deleted-migration check in repo-checks.- migrations_any:- - '**/migrations/*.py'- # Django migration files in any app, plus the routing files that decide- # which app labels land in which database. Both change the plan a- # from-scratch migrate builds. Match everything and exclude the two- # non-Django directories, so a new app is covered without an edit here.- # An app missed by a positive list would leave the replay gate silently- # off for it.- migration_replay:- - '**/migrations/*.py'- - '!posthog/clickhouse/migrations/*.py'- - '!posthog/async_migrations/migrations/*.py'- - 'products/db_routing.yaml'- - 'posthog/product_db_config.py'- - 'posthog/product_db_router.py'- persons_sql:- - 'rust/persons_migrations/*.sql'- - 'rust/bin/migrate-persons'- timing_scripts:- - '.github/scripts/optimize_test_durations.py'- - '.github/scripts/turbo-discover.js'- tasks_temporal:- - 'products/tasks/backend/temporal/**/*'- repo_invariants:- # posthog/test/repo_invariants/ reads AGENTS.md and the semgrep- # rule ids it names. Both are excluded from the backend filter- # (markdown is dropped outright, .semgrep is not listed), so a- # change to either would skip repo-checks and report green while- # leaving a tag pointing at a rule that no longer exists.- # The out-of-band domain ratchet reads these trees, so a resolver added there must run it.- - 'nodejs/src/**'- - 'rust/**'- - 'products/desktop/**'- - 'AGENTS.md'- - '.semgrep/**'- - 'pyproject.toml'- # A generated file in a frontend-only PR still needs a known producer.- - '**/*.generated.*'- - '**/generated/**'- # test_generated_files_are_registered.py also reads the projection- # registry and the ci-python paths filter it must stay in step with.- - 'tools/hogli-commands/hogli_commands/projections.py'- - '.github/workflows/ci-python.yml'- - 'products/ruff.toml'- - '.github/scripts/**'- # test_signals_scout_tool_citations.py checks the MCP tool names that the- # canonical scout skills cite against the committed tool catalog. The backend- # filter matches only the .py files in a skill and nothing under services/mcp/.- # The same test fails when a file it reads has no match in this filter.- - 'products/signals/skills/**'- - 'services/mcp/schema/tool-definitions.json'- - 'services/mcp/schema/generated-tool-definitions.json'- # _EXTERNAL_COMPANION_SKILL_DIRS in products/signals/backend/scout_harness/lazy_seed.py- # names this companion skill from another product.- - 'products/replay_vision/skills/exploring-replay-vision-observations/**'- sdk_manifests:- # Everything the SDK major-version guard reads. Lockfiles are listed- # because a floor-only or `*` spec can move to a new major through- # its lock alone.- - '**/package.json'- - 'pnpm-workspace.yaml'- - 'pnpm-lock.yaml'- - 'products/desktop/pnpm-workspace.yaml'- - 'products/desktop/pnpm-lock.yaml'- - 'pyproject.toml'- - 'uv.lock'- - 'services/llm-gateway/pyproject.toml'- - 'services/llm-gateway/uv.lock'- - 'common/ingestion/acceptance_tests/requirements.txt'- - 'tools/infra-scripts/clitools/requirements.txt'- - 'tools/infra-scripts/clitools/constraints.txt'- - 'rust/Cargo.toml'- - 'rust/Cargo.lock'- - 'cli/Cargo.toml'- - 'cli/Cargo.lock'- - 'bin/hobby-installer/go.mod'- - 'bin/hobby-installer/go.sum'- - '.github/scripts/check_sdk_majors.py'- # The guard scans every package.json for an SDK declaration.- - '**/package.json'- - 'rust/Cargo.toml'- - 'cli/Cargo.toml'- - 'bin/hobby-installer/go.mod'- product_yamls:- # Validates owner team slugs against PostHog/posthog collaborator teams,- # and lints the distributed owners.yaml files (schema, conflicts, dead- # globs, coverage). Costs one GitHub API call; gated here so we only pay- # it when ownership actually changes. Drives the validate-product-yamls job.- - 'products/*/product.yaml'- - '**/owners.yaml'- # Resolver changes must re-run the owners.yaml lint too.- - 'packages/owners-yaml/**'- # Transitional: branches that predate the move still carry the- # resolver at tools/owners.- - 'tools/owners/**'- openapi_types:- # Generated OpenAPI types - validate they match schema- - 'frontend/src/generated/**/*'- - 'frontend/src/lib/agentScopes.generated.ts'- - 'products/*/frontend/generated/**/*'- - 'services/mcp/src/generated/**/*'- - 'services/mcp/src/api/generated.ts'- - 'tools/openapi-codegen/**/*'- - 'services/mcp/scripts/lib/**'- - 'frontend/bin/generate-openapi-types.mjs'- - 'frontend/src/lib/api-orval-mutator.ts'- - 'services/mcp/scripts/**/*'- # MCP tool config sources - feed generate-tools.ts- - 'services/mcp/definitions/**/*.yaml'- - 'services/mcp/definitions/prompts/**/*.md'- - 'products/*/mcp/**/*.yaml'- - 'products/*/mcp/prompts/**/*.md'- - 'services/mcp/schema/tool-definitions.json'- - 'services/mcp/schema/tool-definitions-v2.json'- - 'services/mcp/src/tools/generated/**/*'- - 'services/mcp/schema/generated-tool-definitions.json'- - 'services/mcp/schema/tool-definitions-all.json'- # Exec command reference templates - feed generate-exec-docs.ts- - 'services/mcp/src/templates/sections/**/*'- - 'services/mcp/schema/exec-command-reference.md'- # Desktop API client - regenerated by bin/build-desktop-api-client.sh- - 'products/desktop/packages/api-client/src/generated.ts'- - 'products/desktop/packages/api-client/endpoint-allowlist.json'- # Pins typed-openapi exactly, so a generator bump changes this file and not only the desktop lockfile.- - 'products/desktop/packages/api-client/package.json'- - 'products/desktop/packages/api-client/scripts/**'- - 'bin/build-desktop-api-client.sh'-- detect-snapshot-mode:- name: Detect snapshot mode- runs-on: depot-ubuntu-24.04- timeout-minutes: 5- needs: [changes]- # Master pushes skip the matrices (see the trigger comment); skipping this- # root cascades to every downstream test job via their result checks.- if: needs.changes.outputs.backend == 'true' && github.event_name != 'push'- outputs:- mode: ${{ steps.detect.outputs.mode }}- steps:- - name: Detect mode- id: detect- env:- PR_REPO: ${{ github.event.pull_request.head.repo.full_name }}- REPO: ${{ github.repository }}- HAS_NO_SNAPSHOT_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'no-snapshot-update') }}- AUTHOR: ${{ github.actor }}- run: |- if [ "$PR_REPO" != "$REPO" ] && [ -n "$PR_REPO" ]; then- echo "mode=check" >> $GITHUB_OUTPUT- echo "Fork detected - running in CHECK mode (no commits allowed)"- elif [ "$HAS_NO_SNAPSHOT_LABEL" == "true" ]; then- echo "mode=check" >> $GITHUB_OUTPUT- echo "::notice::🔍 Running in CHECK mode - 'no-snapshot-update' label detected"- else- echo "Workflow triggered by: $AUTHOR"- # Dependabot is excluded - it creates new PRs that may need snapshot updates- # Other bots (github-actions, posthog-bot) commit snapshots and must use CHECK mode to avoid loops- if [[ "$AUTHOR" != "dependabot[bot]" ]] && \- ([[ "$AUTHOR" == *"github-actions"* ]] || [[ "$AUTHOR" == *"[bot]"* ]] || [[ "$AUTHOR" == "posthog-bot" ]]); then- echo "mode=check" >> $GITHUB_OUTPUT- echo "::notice::🔍 Running in CHECK mode - snapshots must match exactly"- else- echo "mode=update" >> $GITHUB_OUTPUT- echo "::notice::🔄 Running in UPDATE mode - snapshots can be updated"- fi- fi-- # Fast pre-job: determines which products need testing and if Django should run- # Only needs pnpm + node — no Python, Docker, or services- # Runs on depot to match the product-test runner environment- turbo-discover:- needs: changes- # Skipped on master push: the merge queue already ran the full matrices for- # every landed commit, and the hourly scheduled run keeps master coverage.- if: needs.changes.outputs.backend == 'true' && github.event_name != 'push'- runs-on: depot-ubuntu-24.04- timeout-minutes: 20- name: Discover product tests- env:- # Whether this run selects backend tests at all: a PR, off the merge queue's- # trunk-merge/** run (that run is the merge gate and stays full), and without- # the run-ci-backend label. turbo-discover.js reads it too, so a run that never- # selects reports an empty mode rather than a selector error.- SELECTION_APPLIES: ${{ github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') && !contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && needs.changes.outputs.backend == 'true' }}- # GitHub tests PRs as synthetic merges. The first parent pins the exact integrated- # base for ordinary, stacked, and merge-queue PRs.- TURBO_SCM_BASE: ${{ github.event_name == 'pull_request' && format('{0}^1', github.sha) || '' }}- outputs:- run_legacy: ${{ steps.discover.outputs.run_legacy }}- run_legacy_reason: ${{ steps.discover.outputs.run_legacy_reason }}- matrix: ${{ steps.discover.outputs.matrix }}- schema_cache_key: ${{ steps.schema-key.outputs.key }}- schema_migrations_key: ${{ steps.schema-key.outputs.migrations_key }}- schema_prefix_key: ${{ steps.schema-key.outputs.prefix_key }}- django_shards: ${{ steps.discover.outputs.django_shards }}- # Which Django tests this PR runs. Empty mode = this run does not select.- mode: ${{ steps.discover.outputs.mode }}- # The rest of that decision (segment file lists, run_poe, run_temporal,- # segment_shards) as one JSON object; consumers read it with fromJSON.- selection: ${{ steps.discover.outputs.selection }}- # The posthog-ci-test-selection event properties, as one JSON object.- telemetry: ${{ steps.discover.outputs.telemetry }}-- steps:- # A deep checkout is slow with or without blob:none: the blobless one fetches every file of- # the tree in a second request. Check out at depth 1, then add the history the merge-base- # steps read, without file contents.- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2-- - name: Fetch history without file contents- # --depth, not --deepen: --deepen adds no history once GitHub has replaced the PR's- # merge ref, because no ref then points at GITHUB_SHA.- run: |- git fetch --no-tags --filter=blob:none --depth=1000 origin "$GITHUB_SHA"- echo "History depth: $(git rev-list --count HEAD)"-- - name: Verify PR merge for test selection- id: verify-merge- if: github.event_name == 'pull_request'- env:- PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}- run: |- set -euo pipefail- trap 'echo "::error::Cannot verify the PR merge checkout. Check the checkout ref and the history fetch before retrying." >&2' ERR- test "$(git rev-parse HEAD)" = "$GITHUB_SHA"- test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3- test "$(git rev-parse HEAD^2)" = "$PR_HEAD_SHA"- git rev-parse --verify "$TURBO_SCM_BASE^{commit}"-- - name: Fetch PR base for schema cache- if: github.event_name == 'pull_request'- env:- BASE_REF: ${{ github.event.pull_request.base.ref }}- run: git fetch --no-tags --depth=1000 --filter=blob:none origin "$BASE_REF:refs/remotes/origin/$BASE_REF"-- - name: Compute schema cache key from merge-base- id: schema-key- if: github.event_name == 'pull_request'- env:- BASE_REF: ${{ github.event.pull_request.base.ref }}- run: |- # HEAD is the synthetic merge commit; HEAD^2 is the PR branch tip.- # The history fetch and the base-ref fetch above ensure the- # full ancestry needed to find the divergence point is available.- MERGE_BASE=$(git merge-base HEAD^2 "origin/${BASE_REF}" 2>/dev/null || echo "")- if [ -z "$MERGE_BASE" ]; then- echo "key=" >> $GITHUB_OUTPUT- echo "migrations_key=" >> $GITHUB_OUTPUT- echo "::notice::merge-base not found (branch too stale?) — schema cache will be skipped"- exit 0- fi- # Routing decides which app labels reach the dump, and the consumers replay them- # from the checked-out copy, so these inputs belong in the key below. A PR that- # edits any of them matches no master dump, the merge-base one included: drop- # both keys and let migrations run from scratch.- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)- DB_ROUTING_HEAD=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)- if [ "$DB_ROUTING" != "$DB_ROUTING_HEAD" ]; then- echo "key=" >> $GITHUB_OUTPUT- echo "migrations_key=" >> $GITHUB_OUTPUT- echo "::notice::db routing config or router differs from the merge base, so the schema cache is skipped"- exit 0- fi- # Exact branch-point key, kept as the fallback restore key below.- echo "key=posthog-schema-master-${MERGE_BASE}" >> $GITHUB_OUTPUT- # Last-resort restore prefix: matches the newest saved dump of any- # migration set, for the window where both exact keys miss. Gated on- # checkout age (a day) because a re-run of an old run keeps its original- # commit: a stale checkout can lack migrations the newest dump records,- # and the migrate top-ups only apply forward, so they cannot repair a- # dump that is newer than the code.- HEAD_AGE_SECONDS=$(( $(date +%s) - $(git show -s --format=%ct HEAD) ))- if [ "$HEAD_AGE_SECONDS" -lt 86400 ]; then- echo "prefix_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-" >> $GITHUB_OUTPUT- fi- # Shared key: hash merge-base schema inputs, not PR head, so PR migrations still- # apply as the pytest delta. uv.lock covers third-party migration changes; the- # postgres image is folded in so a server-version bump invalidates the key.- # Non-Django migration frameworks are excluded from this Postgres schema dump.- # SCHEMA_CACHE_EPOCH (workflow env) prefixes the key as a manual cache-bust knob.- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" \- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \- | grep -vE '/(clickhouse|async_migrations)/migrations/' \- | LC_ALL=C sort) || true- PG_IMAGE=$(git show "${MERGE_BASE}:docker-compose.base.yml" 2>/dev/null \- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')- if [ -z "$MIG_FILES" ]; then- echo "migrations_key=" >> $GITHUB_OUTPUT- echo "::notice::no migration files matched — restore will fall back to the SHA schema key"- else- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)- echo "migrations_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT- fi-- # Dedicated app: every "Mint setup-action GitHub token" step in this file- # draws on posthog-devex-general rather than the posthog-setup-actions app- # the other workflows share. Backend CI is the largest single consumer of- # setup-action calls in the repo, enough to threaten a shared 15k/hr bucket- # at merge peaks, so it gets one to itself.- - name: Mint setup-action GitHub token- id: setup-gh-token- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}- skip-token-revoke: true-- # turbo-discover.js needs Node and the turbo binary, not the installed workspace.- - name: Set up pnpm and Node- uses: ./.github/actions/setup-pnpm- with:- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- # Needs pnpm from the step above.- - name: Install turbo- uses: ./.github/actions/setup-turbo-- # turbo-discover.js runs .github/scripts/tach_map.py under `uv run- # --no-project` for the dependent cascade; the script pins tach, so no- # Python project sync is needed. A missing uv widens the cascade to- # every product rather than failing the job.- - uses: ./.github/actions/setup-uv- with:- enable-cache: false- continue-on-error: true-- - name: Restore test durations from cache- # Shard counts are derived from .test_durations (Amdahl). Restore the newest- # cache (refreshed by ci-backend-update-test-timing.yml several times a day);- # the fallback step below covers a miss.- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: .test_durations- # run_id never matches — forces the prefix restore-key to the newest entry.- key: posthog-test-durations-${{ github.run_id }}- restore-keys: |- posthog-test-durations--- - name: Fall back to the last published durations- # .test_durations is not in the repo. The timing workflow refreshes the- # cache several times a day and publishes the same files as an artifact- # of its own runs. On a cache miss (retention lapsed, timing workflow- # down for two weeks) take the newest artifact; if that is gone too,- # pytest-split splits by count and turbo-discover sizes from file counts- # until the next timing run.- if: ${{ hashFiles('.test_durations') == '' }}- continue-on-error: true- env:- GH_TOKEN: ${{ github.token }}- run: |- run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \- --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')- if [ -z "$run_id" ]; then- echo "::warning::No successful timing run to fall back to, sharding without durations"- exit 0- fi- gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \- || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"-- - name: Restore per-segment test durations from cache- # Also restored here so the snapshot below covers the per-segment plan files.- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: |- .test_durations.core- .test_durations.temporal- key: posthog-segment-durations-${{ github.run_id }}- restore-keys: |- posthog-segment-durations--- - name: Restore the events_json prune manifest from cache- # The timing workflow rebuilds it from the newest hourly recording. A miss- # leaves the file absent, and the events_json rows then run every listed test.- if: github.event_name == 'pull_request'- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: events-schema-prune.json- key: posthog-events-schema-prune-${{ github.run_id }}- restore-keys: |- posthog-events-schema-prune--- - name: Snapshot the sharding plan for this run- # The timing caches float to their newest entry, so a re-run can shard- # differently from attempt 1 and, under "re-run failed jobs", a failing- # test can move to a shard that never re-runs while the gate goes green.- # Every test job applies this snapshot instead, so all attempts shard- # identically. Best-effort (consumers fall back to the floating caches);- # overwrite lets a "re-run all jobs" attempt re-snapshot.- continue-on-error: true- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- with:- name: test-durations-plan- path: |- .test_durations- .test_durations.core- .test_durations.temporal- events-schema-prune.json- include-hidden-files: true- overwrite: true- # The per-segment files and the prune manifest are cache-only and may legitimately be absent.- if-no-files-found: warn- # GitHub allows re-runs for 30 days.- retention-days: 30-- # The backend test selector picks the Django tests this PR needs, and on a legacy- # diff its product list narrows the product matrix too: turbo-discover.js cannot- # see which products a posthog/ or ee/ edit reaches, and the selector can, because- # it follows the import graph into products/. Both matrices read one verdict, so- # they can never disagree about whether the selection is trustworthy.- # Only on a legacy diff: turbo-discover.js trusts the selection only when the paths- # filter saw a legacy edit, and falls back without a selector run otherwise. The- # DISABLE_BACKEND_TEST_SELECTION kill switch skips the run too; the script still- # reports the `disabled` reason from its own env.- - name: Install uv- # Optional setup: a failure here must fall back to the full matrices like a- # selector failure does, not fail the job that gates every test job.- continue-on-error: true- if: env.SELECTION_APPLIES == 'true' && needs.changes.outputs.legacy == 'true' && vars.DISABLE_BACKEND_TEST_SELECTION != 'true'- uses: ./.github/actions/setup-uv- with:- enable-cache: false-- - name: Run the backend test selector- id: select- # A selector failure or hang must not fail this job, which gates every test job- # in the workflow — turbo-discover.js falls back to the full matrices instead.- # The step timeout bounds a hang well inside the job's, so the fallback still- # reaches the matrices rather than taking the whole job down with it.- continue-on-error: true- timeout-minutes: 8- if: env.SELECTION_APPLIES == 'true' && needs.changes.outputs.legacy == 'true' && vars.DISABLE_BACKEND_TEST_SELECTION != 'true'- shell: bash- run: |- set -euo pipefail- uv run tools/snob_backend_test_selection_shadow.py \- --base-ref "$TURBO_SCM_BASE" \- > /tmp/selection.json-- - name: Discover products to test- id: discover- env:- # On pushes to master, always run everything.- # On PRs, use the path filter to detect legacy changes.- LEGACY_CHANGED: ${{ github.event_name != 'pull_request' || needs.changes.outputs.legacy }}- # Empty when the selector did not run or failed, which falls back to the- # full matrices on a ready PR and skips them on a draft.- SELECTION_JSON: ${{ steps.select.outcome == 'success' && '/tmp/selection.json' || '' }}- # Kill switch — set the DISABLE_BACKEND_TEST_SELECTION repo variable to put- # every PR back on the full matrices without a code change.- SELECTION_DISABLED: ${{ vars.DISABLE_BACKEND_TEST_SELECTION || 'false' }}- # A draft has a later ready run to defer to, so an untrusted selection skips- # the heavy matrices there instead of running all of them.- PR_DRAFT: ${{ github.event.pull_request.draft }}- RUN_NEW_EVENTS_SCHEMA: ${{ contains(github.event.pull_request.labels.*.name, 'test-new-events-schema') || github.event_name == 'workflow_dispatch' }}- SCHEMA_CHANGED: ${{ github.event_name != 'pull_request' || needs.changes.outputs.schema }}- TURBO_SCM_HEAD: ${{ github.sha }}- # Kill switch — drop comma-separated products from the matrix; empty = run all.- SKIP_PRODUCT_TESTS: ${{ vars.SKIP_PRODUCT_TESTS || '' }}- run: |- # turbo-discover.js uses Turbo's Git affectedness to detect- # changed products. Non-isolated product changes trigger the- # full suite (all products + Django).- RESULT=$(node .github/scripts/turbo-discover.js)- echo "Result: $RESULT"- echo "matrix=$(echo "$RESULT" | jq -c '.matrix')" >> $GITHUB_OUTPUT- echo "run_legacy=$(echo "$RESULT" | jq -r '.run_legacy')" >> $GITHUB_OUTPUT- echo "run_legacy_reason=$(echo "$RESULT" | jq -r '.run_legacy_reason // empty')" >> "$GITHUB_OUTPUT"- echo "django_shards=$(echo "$RESULT" | jq -c '.django_shards // empty')" >> $GITHUB_OUTPUT- echo "$RESULT" | jq -r '"mode=\(.selection.mode)", "selection=\(.selection | tojson)", "telemetry=\(.telemetry | tojson)"' >> "$GITHUB_OUTPUT"-- - name: Upload selection artifact- # test-selection-verdict reads it instead of running the selector again. An- # upload failure must not fail this job; the verdict regenerates the selection.- continue-on-error: true- if: always() && steps.select.outcome == 'success'- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- with:- name: select-tests-output- path: /tmp/selection.json- if-no-files-found: ignore- retention-days: 14-- build-product-test-matrix:- name: Build product test matrix- needs: [turbo-discover]- if: >-- !cancelled() &&- needs.turbo-discover.result == 'success' &&- needs.turbo-discover.outputs.matrix != '[]' &&- needs.turbo-discover.outputs.matrix != ''- runs-on: depot-ubuntu-24.04- timeout-minutes: 5- outputs:- include: ${{ steps.build.outputs.include }}- steps:- - name: Build product matrix include list- id: build- env:- PRODUCT_MATRIX_JSON: ${{ needs.turbo-discover.outputs.matrix }}- # The new-events-schema variants double the matrix, so they run only on- # opted-in PRs and manual dispatches. Other runs cover the product paths- # in .github/new-events-schema-targets.txt with the Django matrix's- # events_json rows.- RUN_NEW_EVENTS_SCHEMA: ${{ contains(github.event.pull_request.labels.*.name, 'test-new-events-schema') || github.event_name == 'workflow_dispatch' }}- run: |- if [[ "$RUN_NEW_EVENTS_SCHEMA" == "true" ]]; then- schema_modes='[false, true]'- else- schema_modes='[false]'- echo "Skipping new-events-schema variants (RUN_NEW_EVENTS_SCHEMA is off)"- fi- include=$(jq -cn --argjson matrix "$PRODUCT_MATRIX_JSON" --argjson schema_modes "$schema_modes" '- [- $matrix[] as $entry- | $schema_modes[] as $new_events_schema- | $entry + {"new-events-schema": $new_events_schema}- ]- ')- echo "include=$include" >> "$GITHUB_OUTPUT"- echo "Product test matrix size: $(jq -r 'length' <<< "$include")"-- # Runs product tests in parallel — one matrix job per group- # Each job gets its own runner + Docker stack, so no shared DB conflicts- # Small products (< 50 tests) are grouped into a single job to avoid setup overhead- # A status function is required to inspect dependency results after failures.- # Use !cancelled(), not always(), so superseded matrix jobs still receive cancellation.- turbo-tests:- needs: [changes, turbo-discover, detect-snapshot-mode, build-product-test-matrix, get_clickhouse_versions]- if: >-- !cancelled() &&- (github.event.pull_request.draft != true ||- startsWith(github.head_ref, 'trunk-merge/') ||- contains(github.event.pull_request.labels.*.name, 'run-ci-backend')) &&- needs.turbo-discover.result == 'success' &&- needs.build-product-test-matrix.result == 'success' &&- needs.build-product-test-matrix.outputs.include != '[]' &&- needs.build-product-test-matrix.outputs.include != ''- runs-on: depot-ubuntu-24.04- timeout-minutes: 40- name: Product tests (${{ matrix.group }}, events schema ${{ matrix.new-events-schema && 'json' || 'legacy' }})- strategy:- fail-fast: false- matrix:- include: ${{ fromJson(needs.build-product-test-matrix.outputs.include) }}- env:- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}-- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2-- # Start Docker early (before dependency installs) so containers can pull- # images and initialize while we install deps. This matches the pattern- # used by core-tests/django and makes the later wait step negligible.- - name: Clean up data directories- run: |- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true- continue-on-error: true-- - name: Log in to Docker Hub- continue-on-error: true- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0- with:- username: ${{ vars.DOCKERHUB_USER }}- password: ${{ secrets.DOCKERHUB_TOKEN }}-- - name: Start services- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- COMPOSE_PROFILES: temporal,azure- CLICKHOUSE_SERVER_IMAGE: ${{ needs.get_clickhouse_versions.outputs.local_image }}- run: |- cp posthog/user_scripts/latest_user_defined_function.xml docker/clickhouse/user_defined_function.xml- bin/ci-wait-for-docker launch --background --down \- db redis7 clickhouse zookeeper kafka objectstorage \- temporal elasticsearch objectstorage-azure-- # Dedicated app: this product matrix alone is ~half the offloadable- # setup-action volume, enough to threaten any shared 15k/hr bucket.- - name: Mint setup-action GitHub token- id: setup-gh-token- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ vars.GH_APP_POSTHOG_PRODUCT_TESTS_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_PRODUCT_TESTS_PRIVATE_KEY }}- skip-token-revoke: true-- # Sequential, not a `parallel:` block. Each of these writes $GITHUB_PATH,- # and concurrent writes crash the GitHub Actions runner ("Collection was- # modified" or a missing-key error). The runner fails the step even though- # the tool installed fine, so the job dies in setup with nothing to debug.- #- # turbo launches each backend:test script through pnpm, so pnpm and- # Node must stay on PATH even though nothing here reads the workspace.- - name: Set up pnpm and Node- uses: ./.github/actions/setup-pnpm- with:- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- - name: Set up Python- uses: ./.github/actions/setup-python-cached- with:- python-version: 3.14.7- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- # Needs pnpm from the step above.- - name: Install turbo- uses: ./.github/actions/setup-turbo-- - name: Install uv- id: setup-uv- uses: ./.github/actions/setup-uv-- - name: Install SAML dependencies- if: steps.setup-uv.outputs.cache-hit != 'true'- run: sudo rm -f /etc/apt/sources.list.d/*twingate* && sudo apt-get update && sudo apt-get install -y libxml2-dev libxmlsec1-dev libxmlsec1-openssl-- - name: Install sqlx-cli- uses: ./.github/actions/setup-sqlx-cli-- - name: Install Python dependencies- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev-- - name: Add service hostnames to /etc/hosts- run: echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage temporal" | sudo tee -a /etc/hosts-- - name: Set up needed files- run: |- mkdir -p frontend/dist- touch frontend/dist/index.html frontend/dist/layout.html frontend/dist/exporter.html- ./bin/download-mmdb-- - name: Wait for Docker services- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- COMPOSE_PROFILES: temporal,azure- CLICKHOUSE_SERVER_IMAGE: ${{ needs.get_clickhouse_versions.outputs.local_image }}- # objectstorage answers InvalidAccessKeyId until its bootstrap registers identities.- run: bin/ci-wait-for-docker wait objectstorage-- - name: Restore schema cache from master- if: ${{ github.event_name == 'pull_request' && needs.turbo-discover.outputs.schema_cache_key != '' }}- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: schema.sql.gz- # Prefer the shared content key; fall back to the exact merge-base SHA key,- # then to the newest dump of any migration set. Right after a migration- # lands on master both exact keys miss until ci-backend re-dumps, and a- # from-scratch migrate costs ~17 minutes per shard; pytest's keepdb- # migrate tops a slightly stale dump up in seconds instead.- key: ${{ needs.turbo-discover.outputs.schema_migrations_key || needs.turbo-discover.outputs.schema_cache_key }}- restore-keys: |- ${{ needs.turbo-discover.outputs.schema_cache_key }}- ${{ needs.turbo-discover.outputs.schema_prefix_key }}-- - name: Prime test_posthog from cached schema- # Treat cache problems as misses; the migrate step below rebuilds the DB- # whenever this one reports restored=false.- id: prime-schema- if: ${{ github.event_name == 'pull_request' }}- run: |- echo "restored=false" >> "$GITHUB_OUTPUT"- if [ ! -f schema.sql.gz ]; then- echo "::notice::Schema cache miss; the migrate step will build test_posthog"- exit 0- fi- mkdir -p .postgres-backups- mv schema.sql.gz .postgres-backups/schema-latest.sql.gz- if ./bin/hogli db:restore-test-db; then- echo "restored=true" >> "$GITHUB_OUTPUT"- else- echo "::warning::Schema restore failed (stale/incompatible cached dump?); the migrate step will build test_posthog"- fi-- - name: Migrate test_posthog from scratch- # Any run without a restored schema cache (master, dispatches, PR cache- # misses) replays the full migration chain here, in a process that exits- # before tests: the executor's memory high-water stays resident in- # whichever process runs it, and pytest must not carry it through the- # test phase. pytest --reuse-db adopts the DB like the PR prime above;- # any failure falls back to the in-pytest migrate.- if: ${{ github.event_name != 'pull_request' || steps.prime-schema.outputs.restored != 'true' }}- env:- DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog- run: |- admin_psql() { docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres "$@"; }- if ! admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" -c "CREATE DATABASE test_posthog;"; then- echo "::warning::Could not recreate test_posthog; pytest --reuse-db will run the full migrate itself"- exit 0- fi- if ! python manage.py migrate; then- echo "::warning::Out-of-process migrate failed; pytest --reuse-db will run the full migrate itself"- admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" || true- fi-- - name: Register Temporal search attributes- # Some product groups (data_warehouse, batch_exports) connect to a live Temporal- # via sync_connect() in their tests, so keep a readiness gate before tests run.- # By this point Temporal has warmed up in the background, so the wait is negligible.- # data_warehouse tests use custom search attributes (PostHogTeamId, PostHogOrgId,- # PostHogDagId) when scheduling materialization, so they must be registered- # before tests run.- # If the command hangs, SIGABRT makes faulthandler print the Python stack of each- # thread before the process exits, so the log shows where it stopped.- timeout-minutes: 10- run: |- bin/wait-for-docker temporal- PYTHONFAULTHANDLER=1 timeout --signal=ABRT --kill-after=30s 3m python manage.py register_temporal_search_attributes-- - name: Restore test durations from cache- # pytest-split distributes this product's tests across shards using- # .test_durations. Restore the newest cache; the fallback step below covers- # a miss.- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: .test_durations- # run_id never matches — forces the prefix restore-key to the newest entry.- key: posthog-test-durations-${{ github.run_id }}- restore-keys: |- posthog-test-durations--- - name: Fall back to the last published durations- # .test_durations is not in the repo. The timing workflow refreshes the- # cache several times a day and publishes the same files as an artifact- # of its own runs. On a cache miss (retention lapsed, timing workflow- # down for two weeks) take the newest artifact; if that is gone too,- # pytest-split splits by count and turbo-discover sizes from file counts- # until the next timing run.- if: ${{ hashFiles('.test_durations') == '' }}- continue-on-error: true- env:- GH_TOKEN: ${{ github.token }}- run: |- run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \- --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')- if [ -z "$run_id" ]; then- echo "::warning::No successful timing run to fall back to, sharding without durations"- exit 0- fi- gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \- || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"-- - name: Download the run's sharding plan snapshot- # Same pinning as the Django job; falls back to the floating cache.- id: pinned-plan- continue-on-error: true- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- name: test-durations-plan- path: /tmp/test-durations-plan-- - name: Apply the pinned sharding plan- if: steps.pinned-plan.outcome == 'success'- shell: bash- run: |- if [ ! -f /tmp/test-durations-plan/.test_durations ]; then- echo "::warning::Sharding-plan snapshot is incomplete, keeping the floating cache restore"- exit 0- fi- cp /tmp/test-durations-plan/.test_durations .test_durations- echo "Sharding plan pinned to this run's snapshot"-- - name: Warn when a re-run reshards unpinned- if: github.run_attempt != '1' && steps.pinned-plan.outcome != 'success'- shell: bash- run: echo "::warning::Sharding-plan snapshot unavailable, this re-run reshards from the floating timing cache and may not re-execute tests that failed in earlier attempts"-- - name: Run product tests- id: run-product-tests- # continue-on-error so the quarantine gate below is the verdict — it passes when- # every failure is an already-quarantined flake, fails otherwise.- continue-on-error: true- # --force: discover already decided this product needs testing, skip turbo cache- # --log-order=stream: stream pytest output live instead of buffering until completion- # legs: one turbo invocation per entry, each with its own pytest-split flags- # (e.g. "-- --splits 3 --group 1"). A job holds several legs when a split- # product's last shard had budget left over for whole small products, and- # pytest-split flags must not leak from that shard onto its job-mates.- env:- PRODUCT_LEGS: ${{ toJSON(matrix.legs) }}- LEGACY_FILTERS: ${{ matrix.filters }}- LEGACY_PYTEST_ARGS: ${{ matrix.pytest_args }}- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- # Person-on-events v2 reads are the production default for every project- # created since 2024-06-14, so product tests run with the mode on, as the- # Core Django rows do. Without it a product query runner is only ever- # tested in the legacy joined mode, and its snapshots record SQL that no- # recent project executes. Tests that need the legacy mode pin it- # themselves with override_settings.- # marker: product-poe-default. The run block keeps legacy mode if the- # checked-out workflow lacks this marker and its matching snapshots.- PERSON_ON_EVENTS_V2_ENABLED: 'true'- # products/tasks/backend/temporal moved here from the Django Temporal- # segment, and its conftest and workflows talk to Modal. Injected only- # when that directory changed, matching the gate the Django job used.- MODAL_TOKEN_ID: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_ID || '' }}- MODAL_TOKEN_SECRET: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_SECRET || '' }}- AWS_S3_ALLOW_UNSAFE_RENAME: 'true'- # --reuse-db: keep the test database between sequential product runs to avoid- # ClickHouse drop/create race conditions with ReplicatedMergeTree ZK metadata.- # --durations-path: turbo runs products from products/<name>/, so pytest-split- # needs an explicit path back to the repo-root durations file for balanced sharding.- # --junitxml: written relative to each product's dir (turbo's cwd), so every- # product in the group gets its own products/<name>/junit-product.xml.- # On master, also collect timing data for pytest-split sharding.- PYTEST_ADDOPTS: >-- --reuse-db- --durations-path ../../.test_durations- --junitxml=junit-product.xml- ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && '--snapshot-update --snapshot-warn-unused' || '' }}- ${{ github.ref == 'refs/heads/master' && '--store-durations' || '' }}- ${{ github.event_name == 'pull_request' && '--cov=backend --cov-report=' || '' }}- # turbo runs each product's backend:test with the product dir as CWD, so --cov=backend- # scopes coverage to that product and saves its data to products/<name>/.coverage. This works- # for bin-packed buckets (each product writes its own file) and split products, whose data- # the report job combines.- # sysmon: Python 3.12+'s low-overhead coverage backend, ~a few % vs ~20% for the C tracer.- COVERAGE_CORE: sysmon- shell: bash- run: |- set +e- # Enable retries only when the checkout includes retry diagnostics.- if grep -qF '"posthog.runner_name"' posthog/conftest.py; then- export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --force-reruns 1 --reruns-delay 1 -r fEsxXR"- fi- overall=0- # A checkout without the matching snapshots must use legacy mode.- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: product-poe-default" .github/workflows/ci-backend.yml; then- echo "::notice::Checked-out revision predates the person-on-events default, running product tests in legacy mode"- export PERSON_ON_EVENTS_V2_ENABLED=false- fi- # Accept a selector that returns the older flat matrix shape.- if [ -z "$PRODUCT_LEGS" ] || [ "$PRODUCT_LEGS" = "null" ]; then- PRODUCT_LEGS=$(jq -cn --arg f "$LEGACY_FILTERS" --arg a "$LEGACY_PYTEST_ARGS" \- '[{filters: $f, pytest_args: $a}]')- echo "Matrix entry carries no legs; running the legacy single invocation"- fi- leg_count=$(jq 'length' <<< "$PRODUCT_LEGS")- # Neither shape present means the job would pass having run no tests, and- # the rollup gate only reads this step's outcome. Fail loudly instead.- if [ -z "$leg_count" ] || [ "$leg_count" -lt 1 ]; then- echo "::error::No test legs in this matrix entry; refusing to pass without running tests"- exit 1- fi- for ((leg_index = 0; leg_index < leg_count; leg_index++)); do- leg_filters=$(jq -r ".[$leg_index].filters" <<< "$PRODUCT_LEGS")- leg_args=$(jq -r ".[$leg_index].pytest_args" <<< "$PRODUCT_LEGS")- echo "::group::turbo backend:test $leg_filters $leg_args"- # Word splitting is intended here: both variables carry several flags.- # shellcheck disable=SC2086- turbo run backend:test $leg_filters --concurrency=1 --output-logs=full --force --log-order=stream $leg_args- exit_code=$?- echo "::endgroup::"- if [ $exit_code -eq 5 ]; then- echo "No tests collected for this leg, this is expected when splitting tests"- elif [ $exit_code -ne 0 ]; then- overall=$exit_code- fi- done- set -e- exit $overall-- # Stages every product's report under a per-product filename. Pass a report path to- # exclude a product from Trunk quarantine; this step then owns that product's verdict.- - name: Prepare product JUnit for Trunk- id: prepare_product_junit- if: always()- shell: bash- env:- PRODUCT_FILTERS: ${{ matrix.filters }}- run: |- bash .github/scripts/prepare-product-junit-for-trunk.sh \- trunk-junit \- "$PRODUCT_FILTERS"-- # Best-effort Trunk upload (continue-on-error); the "Fail on test failure" step below is- # the verdict, so a Trunk outage can't red a passing shard. Internal PRs only (needs the- # secret).- # TRUNK_UPLOAD_ENABLED is the master kill-switch: when it is not 'true' this gate- # skips, so nothing uploads and no known flakes are quarantined. TRUNK_QUARANTINE_ENABLED- # splits the two — it defaults off, so set it to 'true' to let the verdict step below mask- # known flakes; otherwise any real failure reds the job (quarantining needs upload on too).- - name: Quarantine gate- id: quarantine_gate- continue-on-error: true- if: ${{ !cancelled() && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}- uses: ./.github/actions/trunk-quarantine-gate- with:- junit-paths: trunk-junit/junit-product-*.xml- test-collection-id: C0LrorjW # backend-product- # A native-JSON leg uploads under its own variant, so a failure in that mode- # does not read as a flake of the legacy test with the same id.- variant: ${{ matrix.new-events-schema && 'events-json' || '' }}- previous-step-outcome: ${{ steps.run-product-tests.outcome }}- token: ${{ secrets.TRUNK_API_TOKEN }}-- # Verdict: red a real failure the gate didn't clear as a quarantined flake. != 'success'- # also covers the skipped gate on fork/Dependabot. Sits before the timing-data upload so- # a real failure still skips it, as it did before the test step got continue-on-error.- - name: Fail on test failure- if: ${{ !cancelled() && (steps.prepare_product_junit.outcome == 'failure' || (steps.run-product-tests.outcome == 'failure' && (vars.TRUNK_QUARANTINE_ENABLED != 'true' || steps.quarantine_gate.outcome != 'success'))) }}- shell: bash- run: exit 1-- # Feeds the django_tests gate's per-test failure rollup. Rename each file to carry- # its product so the name survives upload-artifact's least-common-ancestor path- # collapse (same trick as "Stage product coverage" below).- - name: Stage product JUnit- if: always()- shell: bash- run: |- mkdir -p junit-staging- for f in products/*/junit-product.xml; do- [ -f "$f" ] || continue- cp "$f" "junit-staging/junit-product-$(basename "$(dirname "$f")").xml"- done- # Separate artifact prefix from junit-results-backend-* so consumers can select- # product JUnit by download pattern; the product name rides in each inner filename- # (junit-product-<name>.xml). The failure rollup and timing reporter consume both.- - name: Upload product test results- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- if: always()- with:- name: product-junit-results-${{ strategy.job-index }}- path: junit-staging/junit-product-*.xml- if-no-files-found: ignore- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwise-- - name: Upload timing data- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- # Only the legacy-schema variant uploads durations — both schema variants run the- # same test IDs, and two artifacts for the same tests would race in the merge.- # A job whose products have no backend tests writes no JUnit, so it has no fresh- # timings either. Its artifact would hold only the restored map, and the timing- # refresh then rejects the whole Products slice because the timing and JUnit shard- # sets differ.- if: ${{ github.ref == 'refs/heads/master' && !matrix.new-events-schema && hashFiles('junit-staging/junit-product-*.xml') != '' }}- with:- name: timing_data-Products-${{ strategy.job-index }}- path: .test_durations- include-hidden-files: true- retention-days: 2-- - name: Stage product coverage- # Rename each products/<name>/.coverage to <name>.coverage so the product name- # survives upload-artifact's least-common-ancestor path collapse.- if: ${{ always() && github.event_name == 'pull_request' }}- shell: bash- run: |- mkdir -p cov-staging- for f in products/*/.coverage; do- [ -f "$f" ] || continue- product=$(basename "$(dirname "$f")")- cp "$f" "cov-staging/${product}.coverage"- done-- - name: Upload product coverage- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- # always(): a failing test shard still has partial coverage worth reporting.- if: ${{ always() && github.event_name == 'pull_request' }}- with:- name: coverage-products-${{ strategy.job-index }}- path: cov-staging/*.coverage- if-no-files-found: ignore- retention-days: 2- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwise-- - name: Verify new snapshots for flakiness- if: ${{ always() && needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}- shell: bash- # The reruns must write the same schema mode's snapshot file as the main- # run, or the new-schema file removed before each rerun never comes back.- env:- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- run: |- .github/scripts/verify-new-snapshots.sh-- - name: Generate snapshot patch- if: ${{ always() && needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}- shell: bash- env:- NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- run: |- mkdir -p /tmp/patches- git add -N '*.ambr' || true- # Same leg scoping as the django job: a leg may only ship its own- # schema mode's snapshot files (see the comment there).- if [[ "$NEW_EVENTS_SCHEMA" == "true" ]]; then- ambr_pathspec=('*.new_events_schema.ambr')- else- ambr_pathspec=('*.ambr' ':(exclude)*.new_events_schema.ambr')- fi- if ! git diff --quiet -- "${ambr_pathspec[@]}" 2>/dev/null; then- git diff --binary --full-index -- "${ambr_pathspec[@]}" > /tmp/patches/backend-Products-${{ strategy.job-index }}.patch- echo "Generated patch with $(wc -l < /tmp/patches/backend-Products-${{ strategy.job-index }}.patch) lines"- else- echo "No snapshot changes to patch"- fi-- - name: Upload snapshot patch- if: ${{ always() && needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- with:- name: snapshot-patch-Products-${{ strategy.job-index }}- path: /tmp/patches/- if-no-files-found: ignore- retention-days: 1-- # Lightweight repo-wide checks that only need Python + uv (no Docker/DB).- # Consolidates checks that previously each spun up their own runner.- # Every PostHog SDK this repo installs must stay on its pinned major; the release- # bots rewrite manifests across majors, so this is the block. Stdlib only, no venv.- sdk-major-guard:- needs: [changes]- if: needs.changes.outputs.sdk_manifests == 'true'- timeout-minutes: 5- permissions: {}- name: SDK major-version guard- runs-on: ubuntu-24.04- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- - name: Check every SDK stays on its pinned major- run: python3 .github/scripts/check_sdk_majors.py-- repo-checks:- needs: [changes]- # product_yamls: ownership-only PRs still need the owners.yaml lint below.- if: (needs.changes.outputs.backend == 'true' || needs.changes.outputs.product_yamls == 'true' || needs.changes.outputs.repo_invariants == 'true')- timeout-minutes: 10- outputs:- deterministic_failure: ${{ steps.deterministic-failure.outputs.deterministic_failure || 'false' }}- permissions:- contents: read- pull-requests: read # list PR files for the deleted-migration check- name: Repo checks (depot-ubuntu-24.04)- runs-on: depot-ubuntu-24.04- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2-- - name: Mint setup-action GitHub token- id: setup-gh-token- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}- skip-token-revoke: true-- - name: Set up Python- uses: ./.github/actions/setup-python-cached- with:- python-version: 3.14.7- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- - name: Install uv- uses: ./.github/actions/setup-uv- with:- enable-cache: false-- - name: Install Python dependencies- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev-- # Above the boundary: a gh-api flake here is retryable setup, not a- # run-cancelling deterministic failure. The classifier runs below.- - name: Fetch deleted migration paths- id: fetch-deleted-migrations- if: github.event_name == 'pull_request' && needs.changes.outputs.migrations_any == 'true'- env:- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}- REPO: ${{ github.repository }}- PR_NUMBER: ${{ github.event.pull_request.number }}- run: .github/scripts/list-removed-renamed-paths.sh > "$RUNNER_TEMP/deleted-migration-paths.txt"-- # Above the boundary for the same reason. product:lint compares the crossings ledger- # with the merge commit's first parent, so the clone needs one more level of history.- - name: Fetch the pull request base- if: github.event_name == 'pull_request'- run: |- git remote get-url origin >/dev/null 2>&1 || git remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"- git fetch --no-tags --depth=2 origin "$GITHUB_SHA"-- # Boundary between network-dependent setup and the deterministic- # checks below. The "Flag deterministic failure" step keys off this- # step's outcome so a transient setup failure doesn't cancel the run.- - name: Mark setup complete- id: setup-complete- run: echo "Setup complete — any failure below is a deterministic check failure"-- # Below the boundary so a real deletion trips the early-cancel gate (see- # safe-django-migrations.md). Gated on the fetch outcome so a skipped fetch- # skips this too, rather than reading a file it never wrote.- - name: Check deleted migrations- if: steps.fetch-deleted-migrations.outcome == 'success'- run: ./bin/hogli lint:migration-deletions < "$RUNNER_TEMP/deleted-migration-paths.txt"-- - name: Bootstrap scaffold product- run: ./bin/hogli product:bootstrap spline_reticulator --non-interactive-- - name: Lint product structure- env:- CROSSINGS_LEDGER_BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || '' }}- run: ./bin/hogli product:lint --all-- # Distributed owners.yaml tree: schema, product.yaml/owners.yaml- # conflicts, dead rule globs, coverage (warn-only). No --live, so no- # secrets — fork PRs get validated too (the --live slug check runs in- # the fork-gated validate-product-yamls job instead).- - parallel:- - name: Lint owners.yaml files- run: ./bin/hogli owners:lint-- - name: Check version specifiers- run: python .github/scripts/check-version-specifiers.py-- - name: Check IDOR model coverage- run: python .github/scripts/check-idor-model-coverage.py-- - name: Check DWH presentation layer stays source-agnostic- run: python .github/scripts/check-dwh-source-agnostic.py-- - name: Check operator parity- run: python .github/scripts/check-operator-parity.py-- - name: Check module boundaries (tach)- # Two passes: tests are in the interface graph but not the dependency graph.- # A branch that predates the command still excludes tests in tach.toml, so- # the old single pass is the right check there.- run: |- if ./bin/hogli lint:tach --help >/dev/null 2>&1; then- ./bin/hogli lint:tach- else- tach check --dependencies --interfaces- fi-- - name: Check product facade enforcement (import-linter)- run: lint-imports-- # These guards take the whole repo as input (import graph after- # django.setup(), signal registries, repo-wide AST scans), so diff-based- # test selection can't pick them and the Django matrix skips them on- # products-only diffs and drafts. They need no service containers.- # Branches that predate the directory still carry the guards at their- # old paths, so fall back to those rather than failing on a missing arg.- - name: Run repo invariants (whole-repo pytest guards)- run: |- if [ -d posthog/test/repo_invariants ]; then- pytest -p no:cacheprovider posthog/test/repo_invariants- else- pytest -p no:cacheprovider posthog/test/test_startup_import_budget.py posthog/test/test_dataclass_defaults.py- fi-- # Deterministic = a static check above failed after setup completed.- # A retry can't fix it, so this signals the cancellation gate to stop the run.- - name: Flag deterministic failure- id: deterministic-failure- if: failure() && steps.setup-complete.outcome == 'success'- run: echo "deterministic_failure=true" >> "$GITHUB_OUTPUT"-- cancel-backend-on-repo-check-failure:- needs: [repo-checks]- if: >- always() &&- needs.repo-checks.outputs.deterministic_failure == 'true' &&- github.event_name == 'pull_request' &&- github.event.pull_request.head.repo.full_name == github.repository- timeout-minutes: 5- permissions:- contents: read- actions: write- name: Cancel Backend CI after repo check failure- runs-on: ubuntu-24.04- steps:- - name: Cancel run- env:- GH_TOKEN: ${{ github.token }}- GH_REPO: ${{ github.repository }}- run: |- echo "::notice title=Repo checks failed deterministically::Canceling remaining Backend CI jobs. A retry cannot fix this failure; push a fix to start a new run."- gh run cancel ${{ github.run_id }}-- # Validates product.yaml owners against PostHog/posthog collaborator teams.- # Gated on the product_yamls paths filter so we only spend a GitHub API call- # when ownership actually changes. The scoped form (positional product names)- # means a PR can't trip on pre-existing stale ownership in unrelated yamls.- # Skipped for fork PRs — app-token secrets aren't available there, and forks- # can't introduce new product.yaml entries that ship to master without a- # same-repo PR going through this check anyway.- validate-product-yamls:- needs: [changes]- if: >- needs.changes.outputs.product_yamls == 'true' &&- github.event_name == 'pull_request' &&- github.event.pull_request.head.repo.full_name == github.repository- timeout-minutes: 5- name: Validate product.yaml owners (ubuntu-24.04)- runs-on: ubuntu-24.04- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2-- # Default GITHUB_TOKEN can't list /repos/{repo}/teams (needs org-level- # `members: read`). Reuse the assign-reviewers app — same secret, same- # use case. `owner: PostHog` scopes the issued token at the org so- # org-level permissions actually flow through; otherwise the token is- # repo-scoped and the org scope gets dropped.- - name: Get app token- id: app-token- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ secrets.GH_APP_POSTHOG_ASSIGN_REVIEWERS_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_ASSIGN_REVIEWERS_PRIVATE_KEY }}- owner: ${{ github.repository_owner }}- repositories: ${{ github.event.repository.name }}-- # Plain setup-python: this PR-only job runs on ubuntu-24.04, whose cache- # store no master job ever saves the python tool cache to, so the- # setup-python-cached restore would miss on every run.- - name: Set up Python- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version: 3.14.7- # Reuse the app token already minted above instead of a second mint.- token: ${{ steps.app-token.outputs.token || github.token }}-- - name: Install uv- uses: ./.github/actions/setup-uv- with:- enable-cache: false-- - name: Install Python dependencies- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev-- - name: Validate changed product.yaml owners- env:- GH_TOKEN: ${{ steps.app-token.outputs.token }}- CHANGED_FILES: ${{ needs.changes.outputs.product_yamls_files }}- run: |- # CHANGED_FILES is space-separated (paths-filter list-files: escape).- # Each entry is products/<name>/product.yaml — strip to <name>.- names=$(echo "$CHANGED_FILES" | tr ' ' '\n' | sed -n 's|^products/\([^/]*\)/product.yaml$|\1|p' | xargs)- # Drop products whose product.yaml was deleted in this PR (e.g. the- # product moved out of products/) — there are no owners left to validate.- names=$(for n in $names; do [ -f "products/$n/product.yaml" ] && printf '%s ' "$n"; done | xargs)- if [ -z "$names" ]; then- echo "No existing product.yaml files in diff after filtering"- else- ./bin/hogli product:lint:owners $names- fi-- - name: Validate owners.yaml identities against GitHub- env:- GH_TOKEN: ${{ steps.app-token.outputs.token }}- CHANGED_FILES: ${{ needs.changes.outputs.product_yamls_files }}- # Scoped to the diff: unscoped, one stale slug fails every owners.yaml PR.- # xargs re-parses the paths-filter escape format, keeping escaped paths whole.- run: printf '%s' "$CHANGED_FILES" | xargs -r ./bin/hogli owners:lint --live-- # Migration validation.- # This job needs Docker + DB — it checks out master first to run baseline- # migrations, then checks out the PR branch. OpenAPI type generation belongs- # in check-openapi-types below.- check-migrations:- needs: [changes]- # migration_replay is listed because it reaches files the other three miss:- # products/db_routing.yaml is neither Python nor under a backend/ directory, and- # a migration in a future top-level app matches none of them. Without it the- # replay gate could be on while the job that runs it never starts.- if: (needs.changes.outputs.backend == 'true' || needs.changes.outputs.migrations == 'true' || needs.changes.outputs.persons_sql == 'true' || needs.changes.outputs.migration_replay == 'true')- # On master this job replays the full migration history from scratch to- # produce the schema dump every PR job restores, and a merge queue batch that- # changes a Django migration replays it a second time onto an empty database.- # That replay grows with every merged migration, so the limit needs headroom- # over two of them: it is there to cancel a hang, not to cap how long the- # history takes.- timeout-minutes: 60- # The "Publish Migration risk check" step posts the check via the- # posthog-tests app token, which carries its own checks:write- # installation permission — so the check lands in that app's own- # (otherwise-empty) check suite rather than floating under a random- # run in the crowded github-actions suite. The job's GITHUB_TOKEN- # therefore no longer needs checks:write. pull-requests:write is still- # required for the migration sections in the CI report.- permissions:- contents: read- pull-requests: write-- name: Validate migrations- runs-on: depot-ubuntu-24.04- env:- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}-- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- clean: false- - name: Clean up data directories with container permissions- run: |- # Use docker to clean up files created by containers- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true- continue-on-error: true-- - name: Log in to Docker Hub- continue-on-error: true- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0- with:- username: ${{ vars.DOCKERHUB_USER }}- password: ${{ secrets.DOCKERHUB_TOKEN }}-- - name: Stop/Start stack with Docker Compose- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- run: |- bin/ci-wait-for-docker launch --background --down db redis7 clickhouse-- - name: Mint setup-action GitHub token- id: setup-gh-token- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}- skip-token-revoke: true-- - name: Set up Python- uses: ./.github/actions/setup-python-cached- with:- python-version: 3.14.7- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- # Direct install: The job later checks out the PR head, which may predate the local action, so its post step would fail.- - name: Install uv- id: setup-uv- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0- with:- version: '0.13.4'- enable-cache: true- cache-dependency-glob: uv.lock- save-cache: ${{ github.ref == 'refs/heads/master' }}-- - name: Install SAML (python3-saml) dependencies- if: steps.setup-uv.outputs.cache-hit != 'true'- run: |- sudo rm -f /etc/apt/sources.list.d/*twingate*- sudo apt-get update- sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl-- - name: Install sqlx-cli- uses: ./.github/actions/setup-sqlx-cli-- # First running migrations from master, to simulate the real-world scenario- - name: Checkout master- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- ref: master- clean: false-- - name: Install python dependencies for master- run: |- UV_PROJECT_ENVIRONMENT=.venv-master uv sync --frozen --dev --no-install-package pytest-split-- - name: Compute master schema cache key- # Must match the migration-set computation in turbo-discover and the- # push-side save below (inputs, hash, and key prefix). Keyed off the- # master checkout (HEAD here) so a restored dump is exactly the state- # that "Run migrations up to master" would build from scratch.- if: github.event_name != 'push'- id: master-schema-key- run: |- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' HEAD \- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \- | grep -vE '/(clickhouse|async_migrations)/migrations/' \- | LC_ALL=C sort) || true- # Routing decides which app labels are in the dump, so these inputs belong in the key.- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)- PG_IMAGE=$(git show "HEAD:docker-compose.base.yml" 2>/dev/null \- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')- if [ -z "$MIG_FILES" ]; then- echo "key=" >> $GITHUB_OUTPUT- else- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)- echo "key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT- fi-- - name: Restore master schema dump from cache- if: github.event_name != 'push' && steps.master-schema-key.outputs.key != ''- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: schema.sql.gz- key: ${{ steps.master-schema-key.outputs.key }}- # The exact key misses in the minutes between a master migration- # landing and its dump being saved. The PR base commit's dump is- # always older than the checkout, so migrate just tops it up.- restore-keys: |- posthog-schema-master-${{ github.event.pull_request.base.sha }}-- - name: Wait for Docker services- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- run: bin/ci-wait-for-docker wait --only db redis7 clickhouse-- - name: Prime posthog DB from cached schema- # Turns "Run migrations up to master" below into a no-op instead- # of a full migration run.- if: github.event_name != 'push'- run: |- if [ ! -f schema.sql.gz ]; then- echo "::notice::Master schema cache miss — the next step runs all migrations from scratch"- exit 0- fi- # The load is atomic: on failure the transaction rolls back and- # the fallback below recreates an empty DB for a full migrate.- # ensure_migration_defaults re-seeds data-migration rows the- # schema-only dump lacks.- if gunzip -c schema.sql.gz | docker compose -f docker-compose.dev.yml exec -T db psql -q -v ON_ERROR_STOP=1 --single-transaction -U posthog posthog > /dev/null \- && .venv-master/bin/python manage.py ensure_migration_defaults; then- echo "Primed posthog DB from cached master schema"- else- echo "::warning::Cached schema restore failed — recreating empty DB; the next step runs all migrations from scratch"- # Terminate lingering sessions first or DROP DATABASE fails- # (same guard as _recreate_database in hogli's db_schema.py).- docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'posthog' AND pid <> pg_backend_pid();"- docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres -c "DROP DATABASE IF EXISTS posthog;"- docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres -c "CREATE DATABASE posthog;"- fi- rm -f schema.sql.gz-- - name: Run migrations up to master- run: |- # Run Django migrations first (excluding managed=False models)- .venv-master/bin/python manage.py migrate- # Then run persons migrations using sqlx; comment out until we've merged- # DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \- # sqlx database create- # DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \- # sqlx migrate run --source rust/persons_migrations/-- # Now we can consider this PR's migrations- - name: Checkout this PR- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- # For same-repo PRs, checkout the actual branch (not the merge commit)- # so origin/master..HEAD reflects PR files. Fork PRs fall back- # to the default merge commit.- ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') && github.event.pull_request.head.ref || github.ref }}- clean: false-- - name: Install python dependencies for this PR- run: |- UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev-- - name: Check migrations and post SQL to CI report- if: github.event_name == 'pull_request' && needs.changes.outputs.migrations == 'true'- env:- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}- CHANGED_FILES: ${{ needs.changes.outputs.migrations_review_files }}- BASE_SHA: ${{ github.event.pull_request.base.sha }}- run: |- # The PR checkout above uses the PR branch, which on unrebased PRs- # predates this script — skip the CI report section rather than fail.- post_migration_section() {- if [ -f .github/scripts/post-django-migration-section.mjs ]; then- node .github/scripts/post-django-migration-section.mjs "$@"- else- echo "Skipping Django migration section — script not present on this checkout"- fi- }-- # If no migration files changed, exit- if [ -z "$CHANGED_FILES" ]; then- echo "No migration files changed"- echo "No new Django migrations need SQL review." > /tmp/django-migration-sql.md- post_migration_section sql clear ok "no new migrations" /tmp/django-migration-sql.md- exit 0- fi-- if [ -z "$BASE_SHA" ]; then- echo "::warning::BASE_SHA is empty — all changed migrations will be treated as new"- else- # Ensure the base commit is available for comparison- git fetch --no-tags --prune --depth=1 origin "$BASE_SHA" || echo "::warning::Could not fetch base SHA $BASE_SHA — all changed migrations will be shown as new"- fi-- # Initialize comment body for SQL changes- COMMENT_BODY="We've detected new migrations on this PR. Review the SQL output for each migration:\n\n"- HAS_NEW_MIGRATIONS=false- MIGRATION_COUNT=0-- # Process each changed migration file (excluding Rust migrations)- for file in $CHANGED_FILES; do- # Skip Rust migrations as they're handled separately by sqlx- if [[ $file =~ rust/persons_migrations ]]; then- continue- fi- # Full migration name, not the bare numeric prefix: squash- # files (0001_squash_*) share prefixes with the historical- # files they replace, and sqlmigrate errors on an ambiguous- # prefix.- if [[ $file =~ migrations/([0-9]+_[^/]+)\.py$ ]]; then- migration_number="${BASH_REMATCH[1]}"- # Get app name by looking at the directory structure- # For new structure products/user_interviews/backend/migrations, we want user_interviews- # For old structure products/user_interviews/migrations, we want user_interviews- if [[ $file =~ products/([^/]+)/backend/migrations/ ]]; then- app_name="${BASH_REMATCH[1]}"- else- app_name=$(echo $file | sed -E 's|^([^/]+/)*([^/]+)/migrations/.*|\2|')- fi-- # A stacked PR can carry master's migration at this path. Keep a- # conflicting version visible even when master has the same path.- if git cat-file -e "$BASE_SHA:$file" 2>/dev/null || git diff --quiet origin/master HEAD -- "$file"; then- echo "Skipping $file (already exists on the base branch or matches master)"- continue- fi-- # Generated squash files carry replaces= lists thousands- # of names long; their SQL dump is unreviewable and the- # state replay per file adds up across an entire squash- # PR. Detect by content, not by name, so the check can't- # go stale.- if grep -q "replaces = \[" "$file"; then- echo "Skipping $file (squash migration)"- continue- fi-- HAS_NEW_MIGRATIONS=true- MIGRATION_COUNT=$((MIGRATION_COUNT + 1))- echo "Checking migration $migration_number for app $app_name"-- # Get SQL output. A sqlmigrate failure degrades to a- # placeholder — this comment is informational, and the- # real migrate step below still gates the job.- if ! SQL_OUTPUT=$(python manage.py sqlmigrate $app_name $migration_number 2>&1); then- echo "::warning::sqlmigrate failed for $app_name $migration_number — see job log"- SQL_OUTPUT="-- sqlmigrate failed for this migration (see the Validate migrations job log)"- fi-- # Add to comment body- COMMENT_BODY+="#### [\`$file\`](https://github.com/${{ github.repository }}/blob/${{ github.sha }}/$file)\n\`\`\`sql\n$SQL_OUTPUT\n\`\`\`\n\n"- fi- done-- if [ "$HAS_NEW_MIGRATIONS" = false ]; then- echo "No new migrations to show (all changed files already exist on base branch)"- echo "No new Django migrations need SQL review." > /tmp/django-migration-sql.md- post_migration_section sql clear ok "no new migrations" /tmp/django-migration-sql.md- exit 0- fi-- # Add timestamp and commit SHA to SQL changes- TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')- COMMIT_SHA="${{ github.event.pull_request.head.sha }}"- COMMIT_SHORT="${COMMIT_SHA:0:7}"- COMMENT_BODY+="\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"-- # Convert \n into actual newlines- COMMENT_BODY=$(printf '%b' "$COMMENT_BODY")- printf '%s' "$COMMENT_BODY" > /tmp/django-migration-sql.md- if [ "$MIGRATION_COUNT" -eq 1 ]; then- MIGRATION_SUMMARY="1 new migration to review"- else- MIGRATION_SUMMARY="$MIGRATION_COUNT new migrations to review"- fi- post_migration_section sql post warn "$MIGRATION_SUMMARY" /tmp/django-migration-sql.md-- - name: Run migration risk analysis and update CI report- if: github.event_name == 'pull_request'- # Step timeout: an analyzer hang fails THIS step (a failure, which- # !cancelled() still publishes as a definitive verdict) instead of- # running out the 20-min job timeout — a job timeout counts as a- # cancellation, which the publish step now skips, silently stranding- # the consumer. Normal analysis finishes in well under a minute.- timeout-minutes: 5- env:- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}- run: |- # The PR checkout above uses the PR branch, which on unrebased PRs- # predates this script — skip the CI report section rather than fail.- post_migration_section() {- if [ -f .github/scripts/post-django-migration-section.mjs ]; then- node .github/scripts/post-django-migration-section.mjs "$@"- else- echo "Skipping Django migration section — script not present on this checkout"- fi- }-- # Get risk analysis for all unapplied migrations (including third-party).- # --output-json gives downstream consumers (the check-run step- # below, and any other tool) a stable structured shape to read.- set +e # Don't exit immediately on error- RISK_ANALYSIS=$(python manage.py analyze_migration_risk --fail-on-blocked --output-json migration_analysis.json 2>/dev/null)- EXIT_CODE=$?- set -e # Re-enable exit on error-- # Save analysis to file for artifact upload- if [ -n "$RISK_ANALYSIS" ]; then- echo "$RISK_ANALYSIS" > migration_analysis.md- fi-- if [ -n "$RISK_ANALYSIS" ] && echo "$RISK_ANALYSIS" | grep -q "Summary:"; then- # Add timestamp and commit SHA to analysis- TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')- COMMIT_SHA="${{ github.event.pull_request.head.sha }}"- COMMIT_SHORT="${COMMIT_SHA:0:7}"- RISK_COMMENT="We've analyzed your migrations for potential risks.\n\n$RISK_ANALYSIS\n\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"- RISK_COMMENT=$(printf '%b' "$RISK_COMMENT")- printf '%s' "$RISK_COMMENT" > /tmp/django-migration-risk.md- if [ $EXIT_CODE -ne 0 ]; then- post_migration_section risk post fail "blocked migration detected" /tmp/django-migration-risk.md- else- post_migration_section risk post ok "migration analysis complete" /tmp/django-migration-risk.md- fi- elif [ $EXIT_CODE -ne 0 ]; then- echo "Migration risk analysis failed. Check the workflow logs for details." > /tmp/django-migration-risk.md- post_migration_section risk post fail "migration analysis failed" /tmp/django-migration-risk.md- else- echo "No Django migrations need risk analysis." > /tmp/django-migration-risk.md- post_migration_section risk clear ok "no migrations to analyze" /tmp/django-migration-risk.md- fi-- # Fail the job if there were blocked migrations- if [ $EXIT_CODE -ne 0 ]; then- exit $EXIT_CODE- fi-- - name: Get app token for Migration risk check- # Mint a posthog-tests installation token so the check is- # attributed to that app instead of github-actions[bot]. The- # app owns no workflow runs, so its check suite contains only- # this check — it shows up as a standalone, deterministically- # placed check rather than nesting under an arbitrary run in- # the crowded github-actions suite.- #- # Skipped on fork PRs: the app secrets aren't exposed to- # workflows triggered from forks, so the token can't be minted- # there. The analyzer still runs in earlier steps and fails the- # job on Blocked migrations; reviewers can read the verdict in- # the uploaded migration-analysis artifact.- #- # continue-on-error: publishing the check is auxiliary. A- # transient token-mint failure (GitHub API blip, secret- # rotation, app misconfig) must not redden check-migrations- # when migration validation itself passed. On failure the- # token output is empty and the publish step below is skipped.- #- # !cancelled(): a cancelled run was superseded by a newer one that- # publishes the verdict, so don't mint a token here. (${{ }} wraps- # the leading !, a YAML tag indicator.)- if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }}- id: migration-risk-app-token- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }}-- - name: Publish Migration risk check- # Migration risk classification is published as a GitHub check on- # the head commit so it shows up in the PR UI alongside CI checks- # and so any tool that already reads check_runs (review bots,- # branch-protection rules, dashboards) can consume it without- # parsing the comment. The check is a CI feature; consumers and- # the analyzer are decoupled.- #- # Published via the posthog-tests app token (see the token step- # above) so the check lands in that app's own check suite.- #- # Published on success or genuine failure (including an analyzer- # crash, or zero migrations to analyze) so every PR ends up with a- # definitive verdict on its head SHA. Consumers can then treat- # "no completed check yet" purely as "CI hasn't finished" and- # don't need a fallback heuristic.- #- # !cancelled(): a cancelled job skipped the analyzer, leaving no- # migration_analysis.json, so an always() publish would stamp a- # spurious "❌ Analyzer failed". The newer run that superseded it- # publishes the real verdict; genuine crashes still publish here.- #- # Skipped on PRs from forks: the app token can't be minted there- # (secrets are withheld from fork-triggered workflows), so the- # check-runs POST would have no credential. The analyzer itself- # still runs in earlier steps and fails the job on Blocked- # migrations; reviewers can read the verdict in the uploaded- # migration-analysis artifact.- #- # Also skipped when the token step failed: its output is then- # empty and the check-runs POST would 401 and redden the job- # for a reason unrelated to migration validation.- if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && steps.migration-risk-app-token.outputs.token != '' }}- env:- GITHUB_TOKEN: ${{ steps.migration-risk-app-token.outputs.token }}- HEAD_SHA: ${{ github.event.pull_request.head.sha }}- REPO: ${{ github.repository }}- run: |- if [ ! -f migration_analysis.json ]; then- # Analyzer crashed before writing JSON. Publish a failure- # check so stamphog (or any consumer) gets a definitive- # verdict instead of looping in "wait for the check."- echo "No migration_analysis.json — analyzer didn't run; publishing failure check"- jq -n \- --arg name "Migration risk" \- --arg head_sha "$HEAD_SHA" \- '{name: $name, head_sha: $head_sha, status: "completed",- conclusion: "failure",- output: {title: "❌ Analyzer failed",- summary: "<!-- stamphog:v1 [] -->\nMigration analyzer did not produce migration_analysis.json. Re-run the Backend CI job; if it keeps failing, inspect the analyzer step logs."}}' \- | gh api "repos/$REPO/check-runs" --method POST --input -- exit 0- fi-- MAX_LEVEL=$(jq -r '.max_level // "none"' migration_analysis.json)-- case "$MAX_LEVEL" in- "Safe") CONCLUSION=success; TITLE="✅ All migrations safe" ;;- "Needs Review") CONCLUSION=neutral; TITLE="⚠️ Needs review" ;;- "Blocked") CONCLUSION=failure; TITLE="❌ Blocked migrations" ;;- "none"|"null")- # Zero Django migrations to analyze. Publish success so- # PRs that touch only ClickHouse/async/rbac migrations- # (which the analyzer doesn't cover) don't leave- # consumers waiting on a check that would never come.- CONCLUSION=success- TITLE="✅ No Django migrations to analyze"- ;;- *)- echo "Unknown max_level '$MAX_LEVEL' — publishing failure"- CONCLUSION=failure- TITLE="❌ Unknown analyzer output"- ;;- esac-- # Embed the analyzed file paths in the summary as a hidden- # marker. Stamphog parses this to scope its deny-list bypass- # to exactly the files the analyzer classified — heuristics- # over directory names produce false bypasses for unrelated- # systems (ClickHouse, async migrations) that share the- # `migrations/` directory naming.- ANALYZED_PATHS=$(jq -c '[.migrations[].file_path | select(. != null)]' migration_analysis.json 2>/dev/null || echo '[]')- MARKER="<!-- stamphog:v1 ${ANALYZED_PATHS} -->"-- # Truncate the human report to fit the check-run output- # limit (~64KB); the full markdown is also posted in the CI- # report for humans. The marker stays at the top so it's- # never lost to truncation.- REPORT_BODY=$(head -c 59000 migration_analysis.md 2>/dev/null || echo "See the CI report")- SUMMARY=$(printf '%s\n%s' "$MARKER" "$REPORT_BODY")-- jq -n \- --arg name "Migration risk" \- --arg head_sha "$HEAD_SHA" \- --arg conclusion "$CONCLUSION" \- --arg title "$TITLE" \- --arg summary "$SUMMARY" \- '{name: $name, head_sha: $head_sha, status: "completed",- conclusion: $conclusion,- output: {title: $title, summary: $summary}}' \- | gh api "repos/$REPO/check-runs" --method POST --input --- - name: Upload migration analysis artifact- if: always() && github.event_name == 'pull_request'- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- with:- name: migration-analysis- path: |- migration_analysis.md- migration_analysis.json- if-no-files-found: ignore-- - name: Run migrations for this PR- run: |- # Run Django migrations first (excluding managed=False models)- python manage.py migrate- # Then run persons migrations using sqlx- DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \- sqlx migrate run --source rust/persons_migrations/-- - name: Replay the full migration history from scratch- # The steps above prime the database from the cached master schema dump, so- # the migrate that follows only applies the migrations this branch adds.- # That hides a branch whose new migration moves an older one's position in- # the plan, because the older one is already recorded as applied and never- # runs against its new neighbours. A replay onto an empty database is the- # only step that runs the whole history in the order this branch produces.- # Master pushes already replay from scratch, which is where such a break- # surfaces today, one merge too late.- #- # Queue lane only. The trunk-merge/** branch is master plus the batch, so it- # sees the ordering every merged migration produces together; a PR branch- # sees it only while it is current with master. The replay is by far the- # most expensive part of this job, so paying it once per batch rather than- # on every push of every migration branch is what keeps it affordable.- if: github.event_name == 'pull_request' && startsWith(github.head_ref, 'trunk-merge/') && needs.changes.outputs.migration_replay == 'true'- env:- REPLAY_DB: posthog_migration_replay- run: |- set -euo pipefail- docker compose -f docker-compose.dev.yml exec -T db psql -q -U posthog postgres \- -c "DROP DATABASE IF EXISTS ${REPLAY_DB};" \- -c "CREATE DATABASE ${REPLAY_DB};"- if ! DATABASE_URL="postgres://posthog:posthog@localhost:5432/${REPLAY_DB}" python manage.py migrate; then- echo "::error::The migration history does not apply to an empty database. A merge to master fails the same way. Read the traceback above for the migration that stopped, and check whether another migration this branch adds moved it in the plan."- exit 1- fi-- - name: Check migrations- # Skip migration safety check on master push (no migration_files from path filter)- if: github.event_name != 'push'- run: |- DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \- sqlx migrate info --source rust/persons_migrations/- python manage.py makemigrations --check --dry-run-- - name: List this PR's ClickHouse migrations- env:- BASE_SHA: ${{ github.event.pull_request.base.sha }}- run: |- set -euo pipefail- export LC_ALL=C- DIFF_BASE=origin/master- if [ -n "$BASE_SHA" ]; then- if ! git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then- git fetch --no-tags --depth=1 --filter=blob:none origin "$BASE_SHA"- fi- DIFF_BASE="$BASE_SHA"- fi- ch_migrations() {- # Migrations share most of their text, so rename detection would pair this PR's- # migration with one master added later and report it as R, dropping it here.- git diff --no-renames --name-only --diff-filter="$1" "$2"..HEAD -- 'posthog/clickhouse/migrations/[0-9]*' | sort- }- ch_migrations A origin/master > "$RUNNER_TEMP/ch-master-added.txt"- ch_migrations AM origin/master > "$RUNNER_TEMP/ch-master-changed.txt"- # A stacked branch can carry master's migrations, and master can edit one later.- # Only a file that master lacks counts as this PR's addition.- ch_migrations A "$DIFF_BASE" | comm -12 - "$RUNNER_TEMP/ch-master-added.txt" > "$RUNNER_TEMP/ch-migrations-added.txt"- ch_migrations AM "$DIFF_BASE" | comm -12 - "$RUNNER_TEMP/ch-master-changed.txt" > "$RUNNER_TEMP/ch-migrations-changed.txt"- cat "$RUNNER_TEMP/ch-migrations-changed.txt"-- - name: Check CH migrations- run: python manage.py test_ch_migrations_are_safe < "$RUNNER_TEMP/ch-migrations-added.txt"-- - name: Render CH migration SQL per cloud environment- # CH migrations can build different `operations` lists at import time depending on- # `settings.CLOUD_DEPLOYMENT` (e.g. cloud-only Kafka tables). Render the SQL each- # environment will execute so reviewers can spot per-env divergence and verify gated- # branches don't sneak in ON CLUSTER or other forbidden patterns. We render once per- # environment and reuse the output both for the Actions log groups and the CI report- # (environment -> node type(s) -> SQL). Environments that render identically are- # grouped together; if all environments match, the environment level is collapsed.- id: render_ch_sql- if: github.event_name == 'pull_request'- run: |- CHANGED=$(cat "$RUNNER_TEMP/ch-migrations-changed.txt")- if [ -z "$CHANGED" ]; then- echo "No ClickHouse migrations changed."- echo "has_changes=false" >> "$GITHUB_OUTPUT"- exit 0- fi- echo "has_changes=true" >> "$GITHUB_OUTPUT"- echo "migration_count=$(echo "$CHANGED" | grep -c .)" >> "$GITHUB_OUTPUT"- {- echo '<!-- ch-migration-sql -->'- echo '## ClickHouse migration SQL per cloud environment'- echo ''- } > ch_migration_sql_comment.md- mkdir -p ch_sql_env- DEPLOYMENTS=('' US EU DEV)- LABELS=(unset US EU DEV)- # Bucket environments by identical rendered output (parallel indexed arrays;- # avoids bash 4 associative arrays) so divergent SQL prints once.- BUCKET_HASH=()- BUCKET_LABEL=()- BUCKET_FILE=()- for idx in "${!DEPLOYMENTS[@]}"; do- file="ch_sql_env/${idx}.md"- # Render once per environment; the same output feeds both the log group and the comment.- # On failure (e.g. an import error in the migration) write a visible marker rather than- # silently dropping the environment — stderr still shows in the step log for debugging.- if ! CLOUD_DEPLOYMENT="${DEPLOYMENTS[$idx]}" python manage.py print_ch_migration_sql --format markdown ${CHANGED} > "$file"; then- echo '- _⚠️ failed to render SQL for this environment — see the workflow logs_' > "$file"- fi- echo "::group::CLOUD_DEPLOYMENT=${LABELS[$idx]}"- cat "$file"- echo "::endgroup::"- h=$(sha1sum "$file" | cut -d' ' -f1)- found=-1- for b in "${!BUCKET_HASH[@]}"; do- if [ "${BUCKET_HASH[$b]}" = "$h" ]; then found=$b; break; fi- done- if [ "$found" -ge 0 ]; then- BUCKET_LABEL[$found]="${BUCKET_LABEL[$found]}, ${LABELS[$idx]}"- else- BUCKET_HASH+=("$h")- BUCKET_LABEL+=("${LABELS[$idx]}")- BUCKET_FILE+=("$file")- fi- done- if [ "${#BUCKET_HASH[@]}" -eq 1 ]; then- # Every environment renders the same SQL — collapse the environment level.- echo '_Identical across all cloud environments (unset, US, EU, DEV)._' >> ch_migration_sql_comment.md- echo '' >> ch_migration_sql_comment.md- cat "${BUCKET_FILE[0]}" >> ch_migration_sql_comment.md- else- for b in "${!BUCKET_HASH[@]}"; do- echo "- **${BUCKET_LABEL[$b]}**" >> ch_migration_sql_comment.md- sed 's/^/ /' "${BUCKET_FILE[$b]}" >> ch_migration_sql_comment.md- done- fi-- - name: Post CH migration SQL to CI report comment- # Sticky section in the shared CI report comment so pushes don't spam the PR.- # GITHUB_TOKEN is read-only for fork PRs, so only comment on same-repo PRs.- # Skipped on trunk-merge/** branches: merge-queue draft PRs are ephemeral and- # unread, and each section write spends shared GITHUB_TOKEN budget.- if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !startsWith(github.head_ref, 'trunk-merge/')- env:- HAS_CHANGES: ${{ steps.render_ch_sql.outputs.has_changes }}- CH_MIGRATION_COUNT: ${{ steps.render_ch_sql.outputs.migration_count }}- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}- run: |- if [ -f .github/scripts/post-ch-migration-section.mjs ]; then- node .github/scripts/post-ch-migration-section.mjs- node frontend/bin/ci-report/delete-legacy-comments.mjs- else- echo "Skipping CH migration section — script not present on this checkout"- fi-- check-openapi-types:- needs: [changes]- if: (needs.changes.outputs.backend == 'true' || needs.changes.outputs.openapi_types == 'true')- timeout-minutes: 20- outputs:- deterministic_failure: ${{ steps.deterministic-failure.outputs.deterministic_failure || 'false' }}- permissions:- contents: read-- name: Validate OpenAPI types- runs-on: depot-ubuntu-24.04- env:- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}-- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- # For same-repo PRs, checkout the actual branch (not the merge commit)- # so generated OpenAPI types can be committed directly. Fork PRs fall- # back to the default merge commit (auto-commit bails out for forks).- ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') && github.event.pull_request.head.ref || github.ref }}- clean: false-- - name: Clean up data directories with container permissions- run: |- # Use docker to clean up files created by containers- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true- continue-on-error: true-- - name: Log in to Docker Hub- continue-on-error: true- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0- with:- username: ${{ vars.DOCKERHUB_USER }}- password: ${{ secrets.DOCKERHUB_TOKEN }}-- - name: Stop/Start stack with Docker Compose- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- run: |- bin/ci-wait-for-docker launch --background --down db clickhouse-- - name: Mint setup-action GitHub token- id: setup-gh-token- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}- skip-token-revoke: true-- # Tool cache steps inlined rather than the setup-python-cached composite:- # this job checks out the PR head ref, which cannot resolve a local- # action that only exists on master until the branch rebases.- - name: Restore Python tool cache- id: python-toolcache- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: ${{ runner.tool_cache }}/Python/3.14.7- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-3.14.7-- - name: Set up Python- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version: 3.14.7- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- - name: Save Python tool cache- if: github.ref == 'refs/heads/master' && steps.python-toolcache.outputs.cache-hit != 'true'- uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: ${{ runner.tool_cache }}/Python/3.14.7- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-3.14.7-- # Direct install: Generated changes target the PR head, which may predate the local action.- - name: Install uv- id: setup-uv- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0- with:- version: '0.13.4'- enable-cache: true- cache-dependency-glob: uv.lock- save-cache: ${{ github.ref == 'refs/heads/master' }}-- - name: Install SAML (python3-saml) dependencies- if: steps.setup-uv.outputs.cache-hit != 'true'- run: |- sudo rm -f /etc/apt/sources.list.d/*twingate*- sudo apt-get update- sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl-- - name: Install python dependencies- run: |- UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev-- - name: Install pnpm dependencies- uses: ./.github/actions/pnpm-install- with:- install-command: pnpm --filter=@posthog/root --filter=@posthog/frontend... --filter=@posthog/mcp... install --frozen-lockfile- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- # Installs only the desktop API client generator, not the Electron app. The desktop- # workspace defaults to the hoisted linker, which ignores --filter and installs every- # package, so the isolated linker is forced here. A branch that predates the generator- # move skips this step.- - name: Install desktop API client generator- working-directory: products/desktop- run: |- if [ ! -f ../../bin/build-desktop-api-client.sh ]; then- echo "This branch predates the desktop API client regen, so there is nothing to install."- exit 0- fi- pnpm install --frozen-lockfile --filter @posthog/api-client --ignore-scripts --config.node-linker=isolated-- - name: Wait for Docker services- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- run: bin/ci-wait-for-docker wait --only db clickhouse-- - name: Add OpenAPI Problem Matcher- run: echo "::add-matcher::.github/openapi-problem-matcher.json"-- - name: Check and update OpenAPI types- id: openapi-check- env:- EVENT_NAME: ${{ github.event_name }}- HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}- BRANCH: ${{ github.event.pull_request.head.ref }}- HEAD_SHA: ${{ github.event.pull_request.head.sha }}- run: |- if ! ./bin/hogli build:openapi; then- echo ""- echo "::error::OpenAPI schema generation failed."- echo ""- echo "If the failure above mentions \"enum naming encountered a non-optimally"- echo "resolvable collision\" or \"SchemaGenerationError: Failing as requested due"- echo "to warnings\", drf-spectacular found a problem in a serializer/viewset and"- echo "\`--fail-on-warn\` turned it into an error."- echo ""- echo "Diagnose locally:"- echo " python manage.py find_enum_collisions # prints a suggested override entry to add"- echo " hogli build:openapi-schema # to reproduce the full warning set"- echo ""- echo "Where to fix:"- echo " posthog/settings/web.py # ENUM_NAME_OVERRIDES (see comment block at the top)"- echo " /improving-drf-endpoints # invoke this skill for the full serializer/viewset guide"- echo ""- exit 1- fi- pnpm --filter=@posthog/mcp run scaffold-yaml -- --sync-all- if git diff --exit-code; then- echo "OpenAPI types are up to date"- echo "needs-commit=false" >> $GITHUB_OUTPUT- exit 0- fi-- echo "OpenAPI types are out of date"-- # A push to a merge queue branch ends the queue attempt.- if [[ "$BRANCH" == trunk-merge/* ]]; then- echo "::error::OpenAPI types are out of date once this pull request is merged with master: $(git diff --name-only | tr '\n' ' ')"- echo "Merge master into the pull request branch and push. CI regenerates the types on that branch."- exit 1- fi-- # On non-PR builds or fork PRs, fail with instructions- if [ "$EVENT_NAME" != "pull_request" ] || \- [ "$HEAD_REPO" != "PostHog/posthog" ]; then- echo ""- echo "::error::OpenAPI types are out of date!"- echo ""- echo "The TypeScript API types in products/*/frontend/generated/ are auto-generated"- echo "from Django serializers and views. When you modify the backend API, you need"- echo "to regenerate these types."- echo ""- echo "To fix, run locally: hogli build:openapi"- echo "Then commit the updated generated files."- echo ""- echo "More info: https://posthog.com/handbook/engineering/type-system"- echo ""- echo "Questions? #team-devex on Slack"- exit 1- fi-- echo "::notice::Committing updated OpenAPI types to PR branch"-- # Verify branch hasn't advanced since CI started- CURRENT_SHA=$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1)- if [ "$CURRENT_SHA" != "$HEAD_SHA" ]; then- echo "::error::Branch advanced during workflow ($HEAD_SHA -> $CURRENT_SHA) — cannot auto-commit OpenAPI types."- echo ""- echo "OpenAPI types are out of date and could not be auto-committed because the branch"- echo "was updated while this workflow was running. Please run locally and push:"- echo ""- echo " hogli build:openapi"- echo ""- echo "Then commit the updated generated files."- echo "needs-commit=false" >> $GITHUB_OUTPUT- exit 1- fi-- # Commit exactly the files this check flagged. A hand-listed pattern- # silently dropped generated paths it did not cover, so the PR passed on- # an empty commit and every later master push failed this same check.- DRIFTED_FILES=$(git diff --name-only | tr '\n' ' ')- echo "Committing: $DRIFTED_FILES"- echo "drifted-files=$DRIFTED_FILES" >> $GITHUB_OUTPUT- echo "needs-commit=true" >> $GITHUB_OUTPUT-- # Deterministic = the OpenAPI check step itself failed (not a transient- # setup or commit-step failure), so a retry can't fix it. Mirrors repo-checks.- - name: Flag deterministic failure- id: deterministic-failure- if: failure() && steps.openapi-check.outcome == 'failure'- run: echo "deterministic_failure=true" >> "$GITHUB_OUTPUT"-- - name: Get app token for OpenAPI type commits- if: steps.openapi-check.outputs.needs-commit == 'true'- id: openapi-app-token- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }}-- - name: Disable auto-merge before OpenAPI type commit- if: steps.openapi-check.outputs.needs-commit == 'true'- env:- GH_TOKEN: ${{ steps.openapi-app-token.outputs.token }}- PR_NUMBER: ${{ github.event.pull_request.number }}- run: gh pr merge --disable-auto "$PR_NUMBER" || echo "Auto-merge was not enabled"-- - name: Commit OpenAPI types via GitHub API (signed)- if: steps.openapi-check.outputs.needs-commit == 'true'- uses: planetscale/ghcommit-action@25309d8005ac7c3bcd61d3fe19b69e0fe47dbdde # v0.2.20- with:- commit_message: 'chore: update OpenAPI generated types'- repo: ${{ github.repository }}- branch: ${{ github.event.pull_request.head.ref }}- file_pattern: ${{ steps.openapi-check.outputs.drifted-files }}- env:- GITHUB_TOKEN: ${{ steps.openapi-app-token.outputs.token }}-- cancel-backend-on-openapi-check-failure:- needs: [check-openapi-types]- if: >- always() &&- needs.check-openapi-types.outputs.deterministic_failure == 'true' &&- github.event_name == 'pull_request' &&- github.event.pull_request.head.repo.full_name == github.repository- timeout-minutes: 5- permissions:- contents: read- actions: write- name: Cancel Backend CI after OpenAPI check failure- runs-on: ubuntu-24.04- steps:- - name: Cancel run- env:- GH_TOKEN: ${{ github.token }}- GH_REPO: ${{ github.repository }}- run: |- echo "::notice title=OpenAPI check failed deterministically::Canceling remaining Backend CI jobs. A retry cannot fix this failure; push a fix to start a new run."- gh run cancel ${{ github.run_id }}-- build_django_matrix:- name: Build Django matrix- needs: [changes, get_clickhouse_versions, turbo-discover]- if: |- !cancelled() &&- needs.changes.outputs.backend == 'true' &&- needs.get_clickhouse_versions.result == 'success'- runs-on: depot-ubuntu-24.04- timeout-minutes: 5- outputs:- include: ${{ steps.build.outputs.include }}- steps:- - name: Build matrix include list- id: build- env:- OLDEST_SUPPORTED_IMAGE: ${{ needs.get_clickhouse_versions.outputs.oldest_supported_image }}- COMPAT_MATRIX_JSON: ${{ needs.get_clickhouse_versions.outputs.compat_matrix }}- DJANGO_SHARDS_JSON: ${{ needs.turbo-discover.outputs.django_shards }}- MODE: ${{ needs.turbo-discover.outputs.mode }}- RUN_POE: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').run_poe }}- RUN_TEMPORAL: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').run_temporal }}- CORE_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').core_files }}- COMPAT_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').compat_files }}- SEGMENT_SHARDS: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').segment_shards }}- # A trunk-merge/** PR opens as a draft but is the merge gate, and- # turbo-discover deliberately does not select for it. Reporting it as a- # draft here would turn that empty MODE into "skip" and build an- # empty matrix, so the queue would merge having run no Django tests- # at all. Not-draft makes the empty MODE fall through to the full- # matrix, which is what the queue has to run.- IS_DRAFT: ${{ github.event.pull_request.draft && !startsWith(github.head_ref, 'trunk-merge/') }}- IS_PR_RUN: ${{ github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') }}- FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') }}- # The new-events-schema variants double the matrix, so they run only on- # opted-in PRs and manual dispatches. Every other run takes the- # events_json rows, which cover .github/new-events-schema-targets.txt only.- RUN_NEW_EVENTS_SCHEMA: ${{ contains(github.event.pull_request.labels.*.name, 'test-new-events-schema') || github.event_name == 'workflow_dispatch' }}- # Narrowed events_json paths and their shard count. Empty files on a run- # that takes the whole list.- JSON_TARGETS_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_files }}- JSON_TARGETS_SHARDS: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_shards }}- # Same rule as the django job's condition: the suite runs for a legacy- # edit, for a product change that reaches legacy code, and when discovery- # failed, which leaves run_legacy empty. Without it only the events_json- # rows for the product matrix run.- DJANGO_SUITE: ${{ needs.changes.outputs.legacy == 'true' || needs.turbo-discover.outputs.run_legacy != 'false' }}- run: |- # :NOTE: Keep shard counts/group ranges in sync with historical Django matrix tuning.- # Consult #team-devex before changing.-- # A draft PR must never fall back to the full matrix. turbo-discover sets- # MODE to "skip"/"full"/"selected"; an empty MODE on a- # draft means it was cancelled or failed (typically ready_for_review- # superseding it mid-flight). Without this, the draft builds the full- # matrix and holds the per-branch concurrency slot for the whole run, so the- # ready-for-review run — the actual merge gate — queues behind it instead- # of starting. Skip here and defer to that ready run. The run-ci-backend- # label intentionally forces the full matrix on a draft, so honor it.- if [[ -z "$MODE" && "$IS_DRAFT" == "true" && "$FORCE_FULL" != "true" ]]; then- echo "::notice::Draft PR with no trusted Django selection — skipping heavy matrices; the ready-for-review run is the full gate"- MODE="skip"- fi-- if [[ "$MODE" == "skip" ]]; then- # Draft PR where selection couldn't be trusted: skip the heavy- # matrices and defer to the full run on ready for review.- echo "include=[]" >> "$GITHUB_OUTPUT"- echo "Django matrix size: 0 (mode=skip)"- exit 0- fi-- if [[ "$DJANGO_SUITE" != "true" ]]; then- # A product change that reaches no legacy code skips the Django suite.- # The product jobs read the legacy events table only, so the events_json- # rows still run the listed paths of the products in the product matrix.- core="[]"- core_new_events_schema="[]"- core_persons_on_events="[]"- core_persons_on_events_new_schema="[]"- temporal="[]"- temporal_new_events_schema="[]"- compat="[]"- json_targets_count=0- [[ -n "$JSON_TARGETS_FILES" ]] && json_targets_count="${JSON_TARGETS_SHARDS:-1}"- elif [[ "$MODE" == "selected" ]]; then- # Selected mode: run only the segments that have selected files. Shard- # counts come from turbo-discover, which applies its per-shard budget to- # the selected tests' recorded seconds. A wide selection therefore- # spreads over several jobs instead of running to the job timeout in- # one. No counts means one shard per segment.- segment_shards="${SEGMENT_SHARDS:-}"- [[ -z "$segment_shards" ]] && segment_shards='{"core": 1, "poe": 1, "temporal": 1}'-- core_count=0- [[ -n "$CORE_FILES" ]] && core_count=$(jq -r '.core' <<< "$segment_shards")- core=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$core_count" '- [range(1; $n + 1) | {- segment: "Core",- "person-on-events": false,- "new-events-schema": false,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $n,- group: .,- artifact_key: ("core-" + (.|tostring)),- compat: false- }]- ')- core_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$core_count" '- [range(1; $n + 1) | {- segment: "Core",- "person-on-events": false,- "new-events-schema": true,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $n,- group: .,- artifact_key: ("core-events-json-" + (.|tostring)),- compat: false- }]- ')- poe_count=0- [[ "$IS_PR_RUN" != "true" && "$RUN_POE" == "true" ]] && poe_count=$(jq -r '.poe' <<< "$segment_shards")- core_persons_on_events=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$poe_count" '- [range(1; $n + 1) | {- segment: "Core",- "person-on-events": true,- "new-events-schema": false,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $n,- group: .,- artifact_key: ("core-poe-" + (.|tostring)),- compat: false- }]- ')- core_persons_on_events_new_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$poe_count" '- [range(1; $n + 1) | {- segment: "Core",- "person-on-events": true,- "new-events-schema": true,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $n,- group: .,- artifact_key: ("core-poe-events-json-" + (.|tostring)),- compat: false- }]- ')- temporal_count=0- [[ "$RUN_TEMPORAL" == "true" ]] && temporal_count=$(jq -r '.temporal' <<< "$segment_shards")- temporal=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$temporal_count" '- [range(1; $n + 1) | {- segment: "Temporal",- "person-on-events": false,- "new-events-schema": false,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $n,- group: .,- artifact_key: ("temporal-" + (.|tostring)),- compat: false- }]- ')- temporal_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$temporal_count" '- [range(1; $n + 1) | {- segment: "Temporal",- "person-on-events": false,- "new-events-schema": true,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $n,- group: .,- artifact_key: ("temporal-events-json-" + (.|tostring)),- compat: false- }]- ')- # Compat covers the older supported ClickHouse versions over- # CLICKHOUSE_COMPAT_PYTEST_TARGETS. Selection can drop it only when the- # diff touches none of those paths — otherwise a narrowed run would- # silently lose the one signal the full matrix has about old servers.- # One unsharded entry per version, over the selected compat files.- if [[ -n "$COMPAT_FILES" ]]; then- compat=$(jq -cn --argjson compat "${COMPAT_MATRIX_JSON:-[]}" '- [ $compat- | map(."clickhouse-server-image") | unique | to_entries[]- | {- "clickhouse-server-image": .value,- segment: "Core",- "person-on-events": false,- "new-events-schema": false,- "python-version": "3.14.7",- concurrency: 1,- group: 1,- compat: true,- artifact_key: ("compat-" + ((.key + 1)|tostring))- }- ]- ')- else- compat="[]"- fi- # The narrowed events_json paths: selected files under a listed path,- # plus the listed paths of the products in the narrowed product matrix.- json_targets_count=0- [[ -n "$JSON_TARGETS_FILES" ]] && json_targets_count="${JSON_TARGETS_SHARDS:-1}"- else- # Full run: auto-shard counts from turbo-discover (Amdahl's law on .test_durations).- # Falls back to hardcoded defaults if turbo-discover failed or data is missing.- CORE_SHARDS=38- CORE_POE_SHARDS=7- TEMPORAL_SHARDS=7- JSON_TARGETS_FULL_SHARDS=5- if [ -n "$DJANGO_SHARDS_JSON" ] && echo "$DJANGO_SHARDS_JSON" | jq -e '.Core.shards' > /dev/null 2>&1; then- CORE_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r '.Core.shards')- CORE_POE_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r '.CorePOE.shards')- TEMPORAL_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r '.Temporal.shards')- JSON_TARGETS_FULL_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r --argjson fallback "$JSON_TARGETS_FULL_SHARDS" '.JsonTargets.shards // $fallback')- echo "Auto-sharding: Core=$CORE_SHARDS, CorePOE=$CORE_POE_SHARDS, Temporal=$TEMPORAL_SHARDS, JsonTargets=$JSON_TARGETS_FULL_SHARDS"- else- echo "::warning::Django shard auto-sizing unavailable, using defaults: Core=$CORE_SHARDS, CorePOE=$CORE_POE_SHARDS, Temporal=$TEMPORAL_SHARDS, JsonTargets=$JSON_TARGETS_FULL_SHARDS"- fi- # A full run takes every listed events_json path, unless discovery- # shortened the list: a skipped product, or a product-only diff whose- # legacy paths the merge queue runs. An empty list gives 0 rows.- json_targets_count="$JSON_TARGETS_FULL_SHARDS"- if [[ -n "$JSON_TARGETS_SHARDS" ]] && (( JSON_TARGETS_SHARDS < json_targets_count )); then- json_targets_count="$JSON_TARGETS_SHARDS"- echo "::notice::Discovery shortened the events_json list to $json_targets_count shard(s)"- fi-- # The legacy-mode safeguard leg runs only where a regression must not- # land: merge queue and master. Ordinary PR lanes run the- # customer-default mode only, so a PR pays no shards for the legacy- # mode. The queue's trunk-merge/** run is a pull_request event as well,- # so it is exempted by branch. Compat keeps its own legacy-mode rows in- # every lane, because it guards old servers, not this.- if [[ "$IS_PR_RUN" == "true" ]]; then- CORE_POE_SHARDS=0- fi-- core=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_SHARDS" '- [range(1; $shards + 1) | {- segment: "Core",- "person-on-events": false,- "new-events-schema": false,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $shards,- group: .,- artifact_key: ("core-" + (.|tostring)),- compat: false- }]- ')-- core_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_SHARDS" '- [range(1; $shards + 1) | {- segment: "Core",- "person-on-events": false,- "new-events-schema": true,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $shards,- group: .,- artifact_key: ("core-events-json-" + (.|tostring)),- compat: false- }]- ')-- core_persons_on_events=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_POE_SHARDS" '- [range(1; $shards + 1) | {- segment: "Core",- "person-on-events": true,- "new-events-schema": false,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $shards,- group: .,- artifact_key: ("core-poe-" + (.|tostring)),- compat: false- }]- ')-- core_persons_on_events_new_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_POE_SHARDS" '- [range(1; $shards + 1) | {- segment: "Core",- "person-on-events": true,- "new-events-schema": true,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $shards,- group: .,- artifact_key: ("core-poe-events-json-" + (.|tostring)),- compat: false- }]- ')-- temporal=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$TEMPORAL_SHARDS" '- [range(1; $shards + 1) | {- segment: "Temporal",- "person-on-events": false,- "new-events-schema": false,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $shards,- group: .,- artifact_key: ("temporal-" + (.|tostring)),- compat: false- }]- ')-- temporal_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$TEMPORAL_SHARDS" '- [range(1; $shards + 1) | {- segment: "Temporal",- "person-on-events": false,- "new-events-schema": true,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $shards,- group: .,- artifact_key: ("temporal-events-json-" + (.|tostring)),- compat: false- }]- ')-- compat_source="${COMPAT_MATRIX_JSON:-[]}"- compat=$(jq -cn --argjson compat "$compat_source" '- [- $compat- | to_entries[]- | .value + {- segment: "Core",- "person-on-events": false,- "new-events-schema": false,- "python-version": "3.14.7",- compat: true,- artifact_key: ("compat-" + ((.key + 1)|tostring))- }- ]- ')- fi-- if [[ "$RUN_NEW_EVENTS_SCHEMA" != "true" ]]; then- # Without the schema copies, only the events_json rows below refresh- # *.new_events_schema.ambr files, so a module outside- # .github/new-events-schema-targets.txt lets its file go stale.- echo "Skipping new-events-schema variants (RUN_NEW_EVENTS_SCHEMA is off)"- core_new_events_schema="[]"- core_persons_on_events_new_schema="[]"- temporal_new_events_schema="[]"- else- # The doubled matrices exclude some curated paths, such as Dagster tests.- # Discovery keeps only those paths so the other tests do not run a third time.- json_targets_count="${JSON_TARGETS_SHARDS:-0}"- fi-- # The events_json rows: Core jobs over the listed paths with the- # native-JSON events table. They keep Core's person-on-events default.- json_targets=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$json_targets_count" '- [range(1; $n + 1) | {- segment: "Core",- "json-targets": true,- "person-on-events": false,- "new-events-schema": true,- "python-version": "3.14.7",- "clickhouse-server-image": $image,- concurrency: $n,- group: .,- artifact_key: ("json-targets-" + (.|tostring)),- compat: false- }]- ')-- include=$(jq -cn \- --argjson core "$core" \- --argjson core_new_events_schema "$core_new_events_schema" \- --argjson core_persons_on_events "$core_persons_on_events" \- --argjson core_persons_on_events_new_schema "$core_persons_on_events_new_schema" \- --argjson temporal "$temporal" \- --argjson temporal_new_events_schema "$temporal_new_events_schema" \- --argjson compat "$compat" \- --argjson json_targets "$json_targets" \- '$core + $core_new_events_schema + $core_persons_on_events + $core_persons_on_events_new_schema + $temporal + $temporal_new_events_schema + $compat + $json_targets')-- echo "include=$include" >> "$GITHUB_OUTPUT"- echo "Django matrix size: $(jq -r 'length' <<< "$include") (mode=$MODE)"-- django:- needs: [changes, turbo-discover, detect-snapshot-mode, get_clickhouse_versions, build_django_matrix]- # Run legacy pytest if:- # 1. Legacy code directly changed (ee/, posthog/)- # 2. OR product changes affect legacy code (turbo-discover outputs run_legacy=true)- # 3. OR turbo-discover itself failed (conservative: run Django on detection failure)- # 4. OR a product-only change has events_json paths to run. build_django_matrix- # then builds only the events_json rows, because the Django suite does not run.- # The non-empty include guard covers drafts in skip mode and selected runs- # that chose zero files — an empty matrix would otherwise fail the job.- # A status function is required to inspect dependency results after failures.- # Use !cancelled(), not always(), so superseded matrix jobs still receive cancellation.- if: |- !cancelled() &&- needs.changes.outputs.backend == 'true' &&- needs.build_django_matrix.result == 'success' &&- needs.build_django_matrix.outputs.include != '[]' &&- (needs.changes.outputs.legacy == 'true' ||- needs.turbo-discover.outputs.run_legacy == 'true' ||- (needs.turbo-discover.result != 'success' && needs.turbo-discover.result != 'skipped') ||- fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_files != '')- # increase for tmate testing- # 45 for slow Temporal shards on master; keep <= deploy gate wait (container-images-cd.yml)- timeout-minutes: 45-- name: Django tests – ${{ matrix.segment }}${{ matrix.compat && ' compat' || '' }}${{ matrix.json-targets && ' json-targets' || '' }} (persons-on-events ${{ (matrix.segment == 'Core' && !matrix.person-on-events && !matrix.compat) && 'on' || 'off' }}, events schema ${{ matrix.new-events-schema && 'json' || 'legacy' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})- # Runner type is performance-critical — consult #team-devex before changing- runs-on: depot-ubuntu-24.04-- strategy:- fail-fast: false- matrix:- include: ${{ fromJson(needs.build_django_matrix.outputs.include) }}- env:- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}-- steps:- # The default ref, which is the PR merge commit. turbo-discover selects the test files- # from this tree, so a checkout of any other ref can lack a file that the selection lists.- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- fetch-depth: 1- lfs: true- clean: false-- # Put signal-fanout on PATH so the pytest steps below can use it as- # their `shell:` wrapper. GHA's `shell:` field doesn't allow- # ${{ github.workspace }} expansion, so a plain name on PATH is the- # only portable way to reference it.- - name: Install signal-fanout- shell: bash- run: sudo install -m 0755 .github/scripts/signal-fanout /usr/local/bin/signal-fanout-- - name: Clean up data directories with container permissions- run: |- # Use docker to clean up files created by containers- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true- continue-on-error: true-- - name: 'Safeguard: ensure no stray Python modules at product root'- run: |- echo "Checking that products/* only contain backend/, frontend/, or shared/ as Python code roots..."- BAD_FILES=$(find products -maxdepth 2 -type f -name "*.py" ! -path "*/backend/*" ! -name "__init__.py" ! -name "conftest.py" -o -maxdepth 2 -type d -name "migrations" ! -path "*/backend/*")- if [ -n "$BAD_FILES" ]; then- echo "❌ Found Python code or migrations outside backend/:"- echo "$BAD_FILES"- echo "Please move these into the appropriate backend/ folder."- exit 1- fi- echo "✅ No stray Python files or migrations found at product roots."-- # Pre-tests-- # Copies the fully versioned UDF xml file for use in CI testing- - name: Log in to Docker Hub- continue-on-error: true- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0- with:- username: ${{ vars.DOCKERHUB_USER }}- password: ${{ secrets.DOCKERHUB_TOKEN }}-- - name: Stop/Start stack with Docker Compose- shell: bash- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- COMPOSE_PROFILES: temporal,azure- CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image || needs.get_clickhouse_versions.outputs.oldest_supported_image }}- WAIT_FOR_DOCKER_LAUNCH_RETRY_DELAY: 5- run: |- cp posthog/user_scripts/latest_user_defined_function.xml docker/clickhouse/user_defined_function.xml- bin/ci-wait-for-docker launch --background --down-all-profiles-- - name: Add service hostnames to /etc/hosts- shell: bash- run: echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts-- - name: Mint setup-action GitHub token- id: setup-gh-token- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}- skip-token-revoke: true-- # Restore before setup-python so the tool cache is warm when it starts.- - name: Restore Python tool cache- id: python-toolcache- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: ${{ runner.tool_cache }}/Python/${{ matrix.python-version }}- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}-- # Sequential, not a `parallel:` block. Both of these write $GITHUB_PATH,- # and concurrent writes crash the GitHub Actions runner ("Collection was- # modified" or a missing-key error). The runner fails the step even though- # the tool installed fine, so the job dies in setup with nothing to debug.- - name: Set up Python- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version: ${{ matrix.python-version }}- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- - name: Install Rust- if: needs.changes.outputs.backend == 'true'- uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9- with:- toolchain: 1.91.1- components: cargo-- - name: Save Python tool cache- if: github.ref == 'refs/heads/master' && steps.python-toolcache.outputs.cache-hit != 'true'- uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: ${{ runner.tool_cache }}/Python/${{ matrix.python-version }}- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}-- # Direct install: Tests run the explicit PR head, which may predate the local action.- - name: Install uv- id: setup-uv-tests- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0- with:- version: '0.13.4'- enable-cache: true- cache-dependency-glob: uv.lock- save-cache: ${{ github.ref == 'refs/heads/master' }}-- - name: Install SAML (python3-saml) dependencies- if: ${{ needs.changes.outputs.backend == 'true' && steps.setup-uv-tests.outputs.cache-hit != 'true' }}- shell: bash- run: |- sudo rm -f /etc/apt/sources.list.d/*twingate* && sudo apt-get update && sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl-- - name: Install sqlx-cli- if: needs.changes.outputs.backend == 'true'- uses: ./.github/actions/setup-sqlx-cli-- - name: Determine if hogql-parser has changed compared to master- shell: bash- id: hogql-parser-diff- run: |- git fetch --no-tags --prune --depth=1 origin master- changed=$(git diff --quiet HEAD origin/master -- common/hogql_parser/ && echo "false" || echo "true")- echo "changed=$changed" >> $GITHUB_OUTPUT- # A branch that predates the crate has nothing to install.- rust_changed=false- if [[ -f rust/hogql/parser/pyproject.toml ]]; then- crate_version=$(grep -m1 '^version = ' rust/hogql/parser/pyproject.toml | sed -E 's/version = "(.*)"/\1/')- # Publishing is a manual step after merge, so master can carry a crate version that- # the root pin does not name yet. That version builds from source too.- if ! git diff --quiet HEAD origin/master -- rust/hogql/parser/ || ! grep -q "\"hogql-parser-rs==$crate_version\"" pyproject.toml; then- rust_changed=true- fi- fi- echo "rust_changed=$rust_changed" >> $GITHUB_OUTPUT-- # tests would intermittently fail in GH actions- # with exit code 134 _after passing_ all tests- # this appears to fix it- # absolute wild tbh https://stackoverflow.com/a/75503402-- # Cache Qt library packages to reduce network dependency and flakiness- # Using cache-apt-pkgs-action instead of setup-qt-libs to avoid network issues- - name: Cache and install Qt libraries- uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0- with:- packages: libegl1 libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 x11-utils libxcb-cursor0 libopengl0- version: 1.0-- - name: Install Python dependencies- shell: bash- run: |- UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev-- - name: Set up Node.js for canvas builder- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && hashFiles('products/canvas/packages/canvas_builder/package.json') != '' }}- uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0- with:- node-version-file: .nvmrc- token: ${{ steps.setup-gh-token.outputs.token || github.token }}-- - name: Install canvas builder dependencies- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}- shell: bash- run: |- if [[ -f products/canvas/packages/canvas_builder/package.json ]]; then- npm ci --ignore-scripts --omit=dev --no-audit --no-fund --prefix products/canvas/packages/canvas_builder- fi-- - name: Install the working version of hogql-parser- if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.changed == 'true' }}- shell: bash- # This is not cached, because the current HEAD version of hogql-parser must be built if it has- # changed (requirements.txt has the already-published version)- run: |- # Short budget: the `||` mirror is the real fallback for a host that is down.- fetch=(curl --fail --location --retry 3 --retry-all-errors --retry-max-time 30 --connect-timeout 10)- "${fetch[@]}" https://www.antlr.org/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip \- || "${fetch[@]}" https://raw.githubusercontent.com/antlr/website-antlr4/gh-pages/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip- # Check that the downloaded archive is the expected runtime - a security measure- anltr_known_md5sum="c875c148991aacd043f733827644a76f"- antlr_found_ms5sum="$(md5sum antlr4-source.zip | cut -d' ' -f1)"- if [[ "$anltr_known_md5sum" != "$antlr_found_ms5sum" ]]; then- echo "Unexpected MD5 sum of antlr4-source.zip!"- echo "Known: $anltr_known_md5sum"- echo "Found: $antlr_found_ms5sum"- exit 64- fi- unzip antlr4-source.zip -d antlr4-source && cd antlr4-source- cmake .- DESTDIR=out make install- sudo cp -r out/usr/local/include/antlr4-runtime /usr/include/- sudo cp out/usr/local/lib/libantlr4-runtime.so* /usr/lib/- sudo ldconfig- cd ..- pip install ./common/hogql_parser-- - name: Install the working version of hogql-parser-rs- if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.rust_changed == 'true' }}- shell: bash- # rust-py is the production parser, so its tests must run against this branch's- # crate rather than the version pinned in the root pyproject.toml.- run: pip install ./rust/hogql/parser-- - name: Set up needed files- shell: bash- run: |- mkdir -p frontend/dist- touch frontend/dist/index.html- touch frontend/dist/layout.html- touch frontend/dist/exporter.html- ./bin/download-mmdb-- - name: Wait for Docker services- shell: bash- # Read-only health poll for core services only. Can't use --wait here- # because it blocks on ALL project containers including temporal, which- # boots slowly (auto-setup runs DB migrations). Temporal was started in- # background above and will be ready by the time temporal tests run.- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- COMPOSE_PROFILES: temporal,azure- CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image || needs.get_clickhouse_versions.outputs.oldest_supported_image }}- run: bin/ci-wait-for-docker wait-- - name: Restore schema cache from master- if: ${{ github.event_name == 'pull_request' && needs.turbo-discover.outputs.schema_cache_key != '' }}- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: schema.sql.gz- # Prefer the shared content key; fall back to the exact merge-base SHA key,- # then to the newest dump of any migration set. Right after a migration- # lands on master both exact keys miss until ci-backend re-dumps, and a- # from-scratch migrate costs ~17 minutes per shard; pytest's keepdb- # migrate tops a slightly stale dump up in seconds instead.- key: ${{ needs.turbo-discover.outputs.schema_migrations_key || needs.turbo-discover.outputs.schema_cache_key }}- restore-keys: |- ${{ needs.turbo-discover.outputs.schema_cache_key }}- ${{ needs.turbo-discover.outputs.schema_prefix_key }}-- - name: Prime test_posthog from cached schema- # Treat cache problems as misses; the migrate step below rebuilds the DB- # whenever this one reports restored=false.- id: prime-schema- if: ${{ github.event_name == 'pull_request' }}- run: |- echo "restored=false" >> "$GITHUB_OUTPUT"- if [ ! -f schema.sql.gz ]; then- echo "::notice::Schema cache miss; the migrate step will build test_posthog"- exit 0- fi- mkdir -p .postgres-backups- mv schema.sql.gz .postgres-backups/schema-latest.sql.gz- if ./bin/hogli db:restore-test-db; then- echo "restored=true" >> "$GITHUB_OUTPUT"- else- echo "::warning::Schema restore failed (stale/incompatible cached dump?); the migrate step will build test_posthog"- fi-- - name: Migrate test_posthog from scratch- # Any run without a restored schema cache (master, dispatches, PR cache- # misses) replays the full migration chain here, in a process that exits- # before tests: the executor's memory high-water stays resident in- # whichever process runs it, and pytest must not carry it through the- # test phase. pytest --reuse-db adopts the DB like the PR prime above;- # any failure falls back to the in-pytest migrate.- if: ${{ github.event_name != 'pull_request' || steps.prime-schema.outputs.restored != 'true' }}- env:- DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog- run: |- admin_psql() { docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres "$@"; }- if ! admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" -c "CREATE DATABASE test_posthog;"; then- echo "::warning::Could not recreate test_posthog; pytest --reuse-db will run the full migrate itself"- exit 0- fi- if ! python manage.py migrate; then- echo "::warning::Out-of-process migrate failed; pytest --reuse-db will run the full migrate itself"- admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" || true- fi-- - name: Restore test durations from cache- # pytest-split distributes Core/Temporal tests across shards using- # .test_durations. Restore the newest cache; the fallback step below covers- # a miss.- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: .test_durations- # run_id never matches — forces the prefix restore-key to the newest entry.- key: posthog-test-durations-${{ github.run_id }}- restore-keys: |- posthog-test-durations--- - name: Fall back to the last published durations- # .test_durations is not in the repo. The timing workflow refreshes the- # cache several times a day and publishes the same files as an artifact- # of its own runs. On a cache miss (retention lapsed, timing workflow- # down for two weeks) take the newest artifact; if that is gone too,- # pytest-split splits by count and turbo-discover sizes from file counts- # until the next timing run.- if: ${{ hashFiles('.test_durations') == '' }}- continue-on-error: true- env:- GH_TOKEN: ${{ github.token }}- run: |- run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \- --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')- if [ -z "$run_id" ]; then- echo "::warning::No successful timing run to fall back to, sharding without durations"- exit 0- fi- gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \- || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"-- - name: Restore per-segment test durations from cache- # File-granularity sharding plans from a per-segment file (.test_durations.core /- # .test_durations.temporal) scoped to exactly what the segment runs, avoiding the- # union file's cross-segment contamination. These are cache-only as well,- # written by the daily timing-update job on master so PR runs can restore them.- # On a miss the file is simply absent and the "Run … tests" step omits- # --split-plan-path, falling back to scoping the union (still correct, pre-fix- # balance). The posthog-segment-durations- prefix is deliberately distinct from- # the union's posthog-test-durations- so the two never shadow each other.- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: |- .test_durations.core- .test_durations.temporal- key: posthog-segment-durations-${{ github.run_id }}- restore-keys: |- posthog-segment-durations--- - name: Compute hypothesis constants cache key- # hypothesis builds its constants pool of property-test inputs by- # AST-parsing every local module in sys.modules. It caches the result- # in .hypothesis/constants under a hash of each source file, so the- # pool is content-addressed: restoring a stale copy is safe, because a- # changed file reads as a miss and rebuilds. CI otherwise pays the full- # parse on every shard's pytest collection (about 10 s per shard).- id: hyp-constants-key- shell: bash- run: |- # The key includes the installed hypothesis version because the- # entry format is an implementation detail, and rotates weekly so- # entries for deleted or rewritten files expire. The restore prefix- # covers the gap until the first master run of a new week saves.- hyp_version=$(python -c "import hypothesis; print(hypothesis.__version__)")- prefix="posthog-hypothesis-constants-${hyp_version}-${HYPOTHESIS_CONSTANTS_EPOCH}-"- {- echo "key=${prefix}$(date -u +%G-%V)"- echo "restore_prefix=${prefix}"- } >> "$GITHUB_OUTPUT"-- - name: Restore hypothesis constants cache- id: hyp-constants- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- # Constants only. Do NOT add .hypothesis/examples: the example- # database decides which inputs property tests replay, so reusing- # one across runs changes what the tests check.- path: .hypothesis/constants- key: ${{ steps.hyp-constants-key.outputs.key }}- restore-keys: |- ${{ steps.hyp-constants-key.outputs.restore_prefix }}-- - name: Download the run's sharding plan snapshot- # Pin every attempt to the plan turbo-discover snapshotted (rationale on- # that step); a missing snapshot falls back to the floating caches above.- id: pinned-plan- continue-on-error: true- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- name: test-durations-plan- path: /tmp/test-durations-plan-- - name: Apply the pinned sharding plan- if: steps.pinned-plan.outcome == 'success'- shell: bash- run: |- if [ ! -f /tmp/test-durations-plan/.test_durations ]; then- echo "::warning::Sharding-plan snapshot is incomplete, keeping the floating cache restores"- exit 0- fi- # rm first so segment files the snapshot lacks don't survive the floating restore- rm -f .test_durations.core .test_durations.temporal- cp /tmp/test-durations-plan/.test_durations .test_durations- for f in .test_durations.core .test_durations.temporal events-schema-prune.json; do- if [ -f "/tmp/test-durations-plan/$f" ]; then cp "/tmp/test-durations-plan/$f" "$f"; fi- done- echo "Sharding plan pinned to this run's snapshot"-- - name: Warn when a re-run reshards unpinned- if: github.run_attempt != '1' && steps.pinned-plan.outcome != 'success'- shell: bash- run: echo "::warning::Sharding-plan snapshot unavailable, this re-run reshards from the floating timing caches and may not re-execute tests that failed in earlier attempts"-- - name: Determine if --snapshot-update should be on- # UPDATE mode: human commits - update snapshots- # CHECK mode: bot commits (after snapshot update) - verify snapshots match exactly- # safeguard allowlist leg (matrix.person-on-events): always update, because it- # runs a subset under the non-default mode and so does not own snapshots- if: ${{ needs.changes.outputs.backend == 'true' && (needs.detect-snapshot-mode.outputs.mode == 'update' || matrix.person-on-events) }}- shell: bash- # --snapshot-warn-unused: pytest-split shards individual tests across- # runners, so a shard reports other shards' snapshots as "unused". The flag- # downgrades that from a failure to a warning. It does NOT prevent deletion:- # --snapshot-update only spares a snapshot when syrupy matches it to a test- # (by file and name) that did not run on this shard.- run: echo "PYTEST_ARGS=--snapshot-update --snapshot-warn-unused" >> $GITHUB_ENV-- - name: Add snapshot flags for compat subset runs- # Compat runs a subset and doesn't own snapshots (main Django rows do).- # Warn on unused snapshots to avoid version-specific false negatives.- if: ${{ needs.changes.outputs.backend == 'true' && matrix.compat }}- shell: bash- run: echo "PYTEST_ARGS=--snapshot-update --snapshot-warn-unused" >> $GITHUB_ENV-- - name: Warn (don't fail) on unused snapshots for sharded CHECK runs- # CHECK mode (fork PRs, the 'no-snapshot-update' label) runs without- # --snapshot-update, so the steps above don't fire and no snapshot flags- # are set. But pytest-split shards individual tests across runners, so each- # shard only exercises a subset of the snapshots in a given file — every- # snapshot owned by a test that landed on another shard is reported as- # "unused". Under sharding that signal is structurally unreliable, so without- # --snapshot-warn-unused a shard fails on those false positives even when- # every test that ran passed (the snapshot report's "N snapshots unused"- # line). Every non-fork run (UPDATE, poe, compat) already passes the flag;- # this just makes forks symmetric. Trade-off: it downgrades unused snapshots- # to a warning, so genuinely orphaned snapshots go uncaught here — but they- # already do everywhere else, and sharding makes per-shard orphan detection- # impossible regardless. Snapshot *mismatches* and missing snapshots still- # fail, so real value drift is still caught. Mutually exclusive with the- # steps above (mode != update, not poe, not compat).- if: ${{ needs.changes.outputs.backend == 'true' && needs.detect-snapshot-mode.outputs.mode != 'update' && !matrix.person-on-events && !matrix.compat }}- shell: bash- run: echo "PYTEST_ARGS=--snapshot-warn-unused" >> $GITHUB_ENV-- # Tests- - name: Log test environment diagnostics- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}- shell: bash- env:- SHARD_GROUP: ${{ matrix.group }}- SHARD_CONCURRENCY: ${{ matrix.concurrency }}- CLICKHOUSE_IMAGE: ${{ matrix.clickhouse-server-image }}- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- COMMIT_SHA: ${{ github.sha }}- BRANCH_NAME: ${{ github.head_ref || github.ref_name }}- run: |- echo "=== Test Environment Diagnostics ==="- echo "Shard: $SHARD_GROUP/$SHARD_CONCURRENCY"- echo "Python version: $(python --version)"- echo "ClickHouse version: $CLICKHOUSE_IMAGE"- echo "Events schema: $CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA"- echo "Commit: $COMMIT_SHA"- echo "Branch: $BRANCH_NAME"- echo "Runner: $(uname -a)"- echo "Memory: $(free -h | head -2)"- echo "Disk: $(df -h / | tail -1)"- echo "Docker containers:"- docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || true- echo "==================================="-- - name: Run Core tests- id: run-core-tests- # continue-on-error so a test failure doesn't fail the job here; the- # quarantine gate at the end of the job is the verdict — it passes when- # every failure is an already-quarantined flake, fails otherwise.- continue-on-error: true- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}- env:- # Person-on-events v2 reads are the production default for every- # project created since 2024-06-14, so the full Core rows run with the- # mode on. matrix.person-on-events marks the safeguard allowlist leg- # that keeps the legacy joined mode covered, not the mode itself: every- # other condition keyed on it (targets, snapshot ownership, timing- # artifacts) selects the leg. Compat rows run the legacy mode as well,- # as the old-server safeguard.- # marker: person-on-events-default. The run block below greps the- # checked-out workflow for this token to tell a head that carries the- # flip from one that predates it.- PERSON_ON_EVENTS_V2_ENABLED: ${{ !matrix.person-on-events && !matrix.compat && 'true' || 'false' }}- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- MODE: ${{ needs.turbo-discover.outputs.mode }}- CORE_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').core_files }}- POE_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').poe_files }}- COMPAT_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').compat_files }}- JSON_TARGETS: ${{ matrix.json-targets && 'true' || 'false' }}- # The narrowed events_json paths. Empty when the row takes the whole list.- JSON_TARGETS_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_files }}- # devex: measure core (posthog/ee) coverage on PRs to size the gap vs products. sysmon = low overhead.- COVERAGE_CORE: sysmon- # backend-coverage-report skips narrowed runs (patch coverage would read- # as false gaps), so instrumenting one just costs time and uploads an- # artifact nothing reads. The events_json rows rerun tests the legacy- # rows already measure, so they skip it as well.- COV_ARGS: ${{ github.event_name == 'pull_request' && needs.turbo-discover.outputs.mode != 'selected' && !matrix.json-targets && '--cov=posthog --cov=ee --cov-config=.github/coverage-core.cfg --cov-report=' || '' }}- # Wrap bash with signal-fanout so the runner's cancel signal is- # propagated to the entire process tree (incl. pytest). Without- # this, GHA only signals the top-level bash and pytest survives- # until the runner's 10s SIGKILL — which sometimes misses- # detached descendants entirely. Installed earlier on PATH.- shell: 'signal-fanout bash --noprofile --norc -eo pipefail {0}'- run: |- set +e- # Enable retries only when the checkout includes retry diagnostics.- RETRY_ARGS=()- if grep -qF '"posthog.runner_name"' posthog/conftest.py; then- RETRY_ARGS=(--force-reruns 1 --reruns-delay 1)- fi- # A missing --cov-config file aborts pytest, so skip coverage if it is absent.- if [[ -n "$COV_ARGS" && ! -f .github/coverage-core.cfg ]]; then- COV_ARGS=""- fi- # A checkout without the matching test pins must use legacy mode.- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: person-on-events-default" .github/workflows/ci-backend.yml; then- echo "::notice::Checked-out revision predates the person-on-events default, running Core in legacy mode"- export PERSON_ON_EVENTS_V2_ENABLED=false- fi- # Unsupported pytest-split flags abort collection, so probe the installed plugin.- SPLITTING_ALGORITHM=duration_based_chunks- SPLIT_GRANULARITY=()- if python -c "from pytest_split.algorithms import Algorithms; from pytest_split.plugin import PytestSplitFilePlugin; raise SystemExit(0 if 'optimal_chunks' in Algorithms.names() else 1)" 2>/dev/null; then- SPLITTING_ALGORITHM=optimal_chunks- SPLIT_GRANULARITY=(--split-granularity=file)- # Scope the file-mode plan to Core's clean per-segment durations- # (restored from the daily master cache) when the pinned pytest-split- # supports it, so the plan isn't poisoned by the union .test_durations's- # cross-segment tests — see AGENTS.md (CI). Only persons-on-events-off,- # non-compat Core matches the .core run-set; POE-on, compat, and the- # events_json rows run a different/narrower scope, and a cache miss with- # no committed file all fall back to scoping the union.- if [ -f .test_durations.core ] && [[ "${{ matrix.person-on-events }}" != "true" ]] && [[ "${{ matrix.compat }}" != "true" ]] && [[ "$JSON_TARGETS" != "true" ]] && python -c "from pytest_split.plugin import SUPPORTS_SPLIT_PLAN_PATH" 2>/dev/null; then- SPLIT_GRANULARITY+=(--split-plan-path .test_durations.core)- fi- fi- if [[ "$JSON_TARGETS" == "true" ]]; then- # The events_json leg. The job checks out the PR head, and a head without- # the list also predates the JSON-mode test fixes that came before it, so- # its listed tests would fail on code the PR did not touch. The merge- # queue's trunk-merge/** head always has the list, so the gate still runs.- if [[ ! -f .github/new-events-schema-targets.txt ]]; then- echo "::notice::The checked-out revision predates .github/new-events-schema-targets.txt, so the events_json leg runs nothing until this branch rebases onto master"- exit 0- fi- # A narrowed run hands over its paths, and every other run takes the- # whole list. Parse it the way turbo-discover.js does.- if [[ -n "$JSON_TARGETS_FILES" ]]; then- read -r -a json_candidates <<< "$JSON_TARGETS_FILES"- else- mapfile -t json_candidates < <(sed -e 's/#.*//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's:/*$::' -e '/^$/d' .github/new-events-schema-targets.txt)- fi- # The PR head can lack a path that the merged list names. pytest stops- # with a usage error on a missing path, so skip that path and say so.- json_targets=()- for target in "${json_candidates[@]}"; do- if [[ -e "$target" ]]; then- json_targets+=("$target")- else- echo "::warning::events_json path $target does not exist in this checkout, so it does not run"- fi- done- if [[ ${#json_targets[@]} -eq 0 ]]; then- exit 0- fi- JSON_SPLIT=()- if [[ "${{ matrix.concurrency }}" -gt 1 ]]; then- JSON_SPLIT=(--splits "${{ matrix.concurrency }}" --group "${{ matrix.group }}"- --splitting-algorithm="$SPLITTING_ALGORITHM" "${SPLIT_GRANULARITY[@]}")- fi- if [[ "$GITHUB_EVENT_NAME" == "pull_request" && "$GITHUB_HEAD_REF" != trunk-merge/* && -f events-schema-prune.json ]]; then- # Skips the tests that a recording showed never read the JSON tables. The- # merge queue runs every listed test, so a skipped test whose code starts- # to read those tables still fails before the change reaches master.- export POSTHOG_EVENTS_SCHEMA_PRUNE_MANIFEST=events-schema-prune.json- fi- # No --store-durations, because the timing data describes legacy-table runs only.- # shellcheck disable=SC2086- pytest -v --tb=short --reuse-db -o junit_duration_report=call "${json_targets[@]}" -m "not async_migrations" \- "${JSON_SPLIT[@]}" \- "${RETRY_ARGS[@]}" -r fEsxXR \- --junitxml=junit-core.xml \- $COV_ARGS \- $PYTEST_ARGS- elif [[ "$MODE" == "selected" ]]; then- # Selected mode: run only the test files chosen by the snob shadow selector.- # Sharded only when build_django_matrix sized the segment above one shard,- # no --store-durations (don't pollute timing data with a partial run).- # One pytest retry overrides product/marker budgets; Engineering analytics- # retains the retry count and failed-attempt diagnostics even when it passes.- if [[ "${{ matrix.compat }}" == "true" ]]; then- targets="$COMPAT_FILES"- elif [[ "${{ matrix.person-on-events }}" == "true" ]]; then- targets="$POE_FILES"- else- targets="$CORE_FILES"- fi- # A selection wide enough to need more than one shard is split like a- # full run. Item granularity on purpose: the target list is already- # narrow, so pre-import file skipping saves nothing, and the per-segment- # plan file describes a different run-set than the selected subset.- SELECTED_SPLIT=()- if [[ "${{ matrix.concurrency }}" -gt 1 ]]; then- SELECTED_SPLIT=(--splits "${{ matrix.concurrency }}" --group "${{ matrix.group }}"- --splitting-algorithm="$SPLITTING_ALGORITHM")- fi- # shellcheck disable=SC2086- pytest -v --tb=short --reuse-db -o junit_duration_report=call $targets -m "not async_migrations" \- "${SELECTED_SPLIT[@]}" \- "${RETRY_ARGS[@]}" -r fEsxXR \- --junitxml=junit-core.xml \- $COV_ARGS \- $PYTEST_ARGS- else- full_targets="posthog ee/"- full_ignores=()- if [[ "${{ matrix.compat }}" == "true" ]]; then- full_targets="$CLICKHOUSE_COMPAT_PYTEST_TARGETS"- elif [[ "${{ matrix.person-on-events }}" == "true" ]]; then- full_targets="./posthog/clickhouse/ ./posthog/api/test/dashboards/test_dashboard.py ee/clickhouse/"- full_ignores+=(--ignore=posthog/hogql_queries --ignore=posthog/hogql)- fi- # shellcheck disable=SC2086- pytest -v --tb=short --reuse-db -o junit_duration_report=call $full_targets -m "not async_migrations" \- --ignore=posthog/temporal \- --ignore=posthog/dags \- --ignore=common/hogvm/python/test \- --ignore=posthog/test/repo_invariants \- "${full_ignores[@]}" \- --splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \- --durations=1000 --durations-min=1.0 --store-durations \- --pytest-durations=100 \- --splitting-algorithm="$SPLITTING_ALGORITHM" \- "${SPLIT_GRANULARITY[@]}" \- "${RETRY_ARGS[@]}" -r fEsxXR \- --junitxml=junit-core.xml \- $COV_ARGS \- $PYTEST_ARGS- fi- exit_code=$?- set -e- if [ $exit_code -eq 5 ]; then- echo "No tests collected for this shard, this is expected when splitting tests"- exit 0- else- exit $exit_code- fi-- - name: Upload core coverage- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- if: ${{ always() && github.event_name == 'pull_request' && needs.turbo-discover.outputs.mode != 'selected' && matrix.segment == 'Core' && !matrix.json-targets }}- with:- name: coverage-core-${{ strategy.job-index }}- path: .coverage- include-hidden-files: true- if-no-files-found: ignore- retention-days: 2- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwise-- # Uncomment this code to create an ssh-able console so you can debug issues with github actions- # (Consider changing the timeout in ci-backend.yml to have more time)- # - name: Setup tmate session- # if: failure()- # uses: mxschmitt/action-tmate@v3-- - name: Log test environment diagnostics- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}- shell: bash- env:- SHARD_GROUP: ${{ matrix.group }}- SHARD_CONCURRENCY: ${{ matrix.concurrency }}- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- COMMIT_SHA: ${{ github.sha }}- BRANCH_NAME: ${{ github.head_ref || github.ref_name }}- run: |- echo "=== Test Environment Diagnostics ==="- echo "Shard: $SHARD_GROUP/$SHARD_CONCURRENCY"- echo "Python version: $(python --version)"- echo "Events schema: $CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA"- echo "Commit: $COMMIT_SHA"- echo "Branch: $BRANCH_NAME"- echo "Runner: $(uname -a)"- echo "Memory: $(free -h | head -2)"- echo "Disk: $(df -h / | tail -1)"- echo "Docker containers:"- docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || true- echo "==================================="-- - name: Run Temporal tests- id: run-temporal-tests- # continue-on-error: the quarantine gate at the end of the job is the verdict.- # See "Run Core tests".- continue-on-error: true- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}- # See "Run Core tests" for why signal-fanout wraps bash here.- shell: 'signal-fanout bash --noprofile --norc -eo pipefail {0}'- env:- AWS_S3_ALLOW_UNSAFE_RENAME: 'true'- MODAL_TOKEN_ID: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_ID || '' }}- MODAL_TOKEN_SECRET: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_SECRET || '' }}- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- MODE: ${{ needs.turbo-discover.outputs.mode }}- TEMPORAL_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').temporal_files }}- run: |- set +e- # Open PR heads must have retry diagnostics before blanket retries are enabled.- RETRY_ARGS=()- if grep -qF '"posthog.runner_name"' posthog/conftest.py; then- RETRY_ARGS=(--force-reruns 1 --reruns-delay 1)- fi- # Pick the split algorithm and granularity at runtime: optimal_chunks +- # file granularity only when the installed pytest-split offers them, else- # duration_based_chunks / item. A workflow edit runs merged with master, so- # an unrebased PR on older pytest-split must not hit an unknown- # --splitting-algorithm / --split-granularity. See AGENTS.md (CI).- SPLITTING_ALGORITHM=duration_based_chunks- SPLIT_GRANULARITY=()- if python -c "from pytest_split.algorithms import Algorithms; from pytest_split.plugin import PytestSplitFilePlugin; raise SystemExit(0 if 'optimal_chunks' in Algorithms.names() else 1)" 2>/dev/null; then- SPLITTING_ALGORITHM=optimal_chunks- SPLIT_GRANULARITY=(--split-granularity=file)- # No --split-plan-path here for now: the cached plan describes- # master's Temporal run-set, which still contains the batch-exports- # and tasks temporal suites this branch moved to their product jobs.- # Cuts drawn around those phantom files starve two shards and- # overload the third (observed 4/6/20 min on a 3-shard split).- # Scoping the union instead intersects with what actually collects.- # Restore the plan path after the timing workflow has rebuilt- # .test_durations.temporal from post-merge junit.- :- fi- if [[ "$MODE" == "selected" ]]; then- # Selected mode: no --store-durations (don't pollute timing data with a- # partial run). The same single-retry budget applies to selected and full runs.- # A selection wide enough to need more than one shard is split like a- # full run. Item granularity on purpose: the target list is already- # narrow, so pre-import file skipping saves nothing, and the per-segment- # plan file describes a different run-set than the selected subset.- SELECTED_SPLIT=()- if [[ "${{ matrix.concurrency }}" -gt 1 ]]; then- SELECTED_SPLIT=(--splits "${{ matrix.concurrency }}" --group "${{ matrix.group }}"- --splitting-algorithm="$SPLITTING_ALGORITHM")- fi- # shellcheck disable=SC2086- pytest -v --tb=short --reuse-db -o junit_duration_report=call $TEMPORAL_FILES -m "not async_migrations" \- "${SELECTED_SPLIT[@]}" \- "${RETRY_ARGS[@]}" -r fEsxXR \- --junitxml=junit-temporal.xml \- $PYTEST_ARGS- else- # No per-test pytest --timeout; rely on the job-level timeout-minutes- # as the single safety net. The per-test guillotine added nothing- # over it (--timeout-method=thread os._exit()s the whole process- # anyway, so the shard dies either way) and was the reason the- # transaction=True overhead in async tests looked like "the- # timeout problem" instead of a per-test cost worth fixing.- pytest -v --tb=short --reuse-db -o junit_duration_report=call posthog/temporal products/signals/backend/emission -m "not async_migrations" \- --splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \- --durations=100 --durations-min=1.0 --store-durations \- --pytest-durations=100 \- --splitting-algorithm="$SPLITTING_ALGORITHM" \- "${SPLIT_GRANULARITY[@]}" \- "${RETRY_ARGS[@]}" -r fEsxXR \- --junitxml=junit-temporal.xml \- $PYTEST_ARGS- fi- exit_code=$?- set -e- if [ $exit_code -eq 5 ]; then- echo "No tests collected for this shard, this is expected when splitting tests"- exit 0- else- exit $exit_code- fi-- - name: Save hypothesis constants cache- # One Core shard per master run writes the weekly entry: a save from- # every shard would race on the same key and churn the shared cache- # budget. cache-hit != 'true' skips the save once this week's entry- # exists, which keeps later master runs cheap.- # continue-on-error: overlapping master runs at week rollover fail its- # reservation check when an earlier run has saved the same key, and a- # cache miss must never red a green run.- continue-on-error: true- if: |- github.ref == 'refs/heads/master' &&- matrix.segment == 'Core' &&- matrix.group == 1 &&- !matrix.person-on-events &&- !matrix.new-events-schema &&- !matrix.compat &&- steps.hyp-constants.outputs.cache-hit != 'true'- uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3- with:- path: .hypothesis/constants- key: ${{ steps.hyp-constants-key.outputs.key }}-- # Post tests- - name: Show docker compose logs on failure- if: failure() && (needs.changes.outputs.backend == 'true' && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure')- shell: bash- run: docker compose -f docker-compose.dev.yml logs-- - name: Upload updated timing data as artifacts- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- # Upload on master, or on PRs that change timing/sharding scripts (for validation)- if: ${{ (github.ref == 'refs/heads/master' || needs.changes.outputs.timing_scripts == 'true') && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.new-events-schema && matrix.clickhouse-server-image == needs.get_clickhouse_versions.outputs.oldest_supported_image }}- with:- name: timing_data-${{ matrix.segment }}-${{ matrix.group }}- path: .test_durations- include-hidden-files: true- retention-days: 2-- - name: Verify new snapshots for flakiness- # Only in UPDATE mode - CHECK mode doesn't update snapshots- if: ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.compat && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure' }}- shell: bash- # The reruns must write the same schema mode's snapshot file as the main- # run, or the new-schema file removed before each rerun never comes back.- env:- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- run: |- .github/scripts/verify-new-snapshots.sh-- - name: Generate snapshot patch- # Only in UPDATE mode - CHECK mode verifies snapshots match exactly- if: ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.compat && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure' }}- shell: bash- env:- NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}- run: |- mkdir -p /tmp/patches- # Stage any new/modified .ambr files so they appear in git diff- git add -N '*.ambr' || true- # Each schema leg may only ship its own snapshot files. A leg still- # *claims* the other schema mode's snapshot names without writing them,- # so syrupy deletes them from the working tree as unused; scoping the- # patch keeps those deletions out of the committed diff.- if [[ "$NEW_EVENTS_SCHEMA" == "true" ]]; then- ambr_pathspec=('*.new_events_schema.ambr')- else- ambr_pathspec=('*.ambr' ':(exclude)*.new_events_schema.ambr')- fi- # Generate patch if there are changes- if ! git diff --quiet -- "${ambr_pathspec[@]}" 2>/dev/null; then- git diff --binary --full-index -- "${ambr_pathspec[@]}" > /tmp/patches/backend-${{ matrix.artifact_key }}.patch- echo "Generated patch with $(wc -l < /tmp/patches/backend-${{ matrix.artifact_key }}.patch) lines"- else- echo "No snapshot changes to patch"- fi-- - name: Upload snapshot patch- # Only in UPDATE mode- if: ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.compat && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure' }}- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- with:- name: snapshot-patch-${{ matrix.artifact_key }}- path: /tmp/patches/- if-no-files-found: ignore- retention-days: 1-- - name: Archive email renders- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- if: needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && !matrix.person-on-events && !matrix.new-events-schema && !matrix.compat- with:- name: email_renders-${{ github.sha }}-${{ github.run_attempt }}-${{ matrix.segment }}-${{ matrix.person-on-events }}-${{ matrix.group }}- path: posthog/tasks/test/__emails__- retention-days: 1- - name: Upload test results- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- if: always()- with:- # Re-run attempts suffix the name: a same-name upload supersedes attempt 1's- # artifact, and the timing reporter needs both to pair a re-run recovery- # (attempt-N pass) with the attempt-1 failure it proves flaky.- name: junit-results-backend-${{ matrix.artifact_key }}${{ github.run_attempt != '1' && format('-attempt{0}', github.run_attempt) || '' }}- path: |- junit-*.xml- !junit-*-retry-failures.xml-- # The helper sets TRUNK_CODEOWNERS_PATH so Trunk attributes uploaded tests to their owners.- - name: Build the Trunk ownership map- if: ${{ !cancelled() && needs.changes.outputs.backend == 'true' && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}- shell: bash- run: |- script=.github/scripts/trunk-codeowners.sh- if [ -x "$script" ]; then "$script" "$RUNNER_TEMP/trunk-codeowners" "junit-*.xml"; fi-- # Best-effort Trunk upload (continue-on-error); the "Fail on test failure" step below is- # the verdict, so a Trunk outage can't red a passing shard. Internal PRs only (needs the- # secret); cli-version pinned, not 'latest' (skips the release redirect).- # TRUNK_UPLOAD_ENABLED is the master kill-switch: when it is not 'true' this gate- # skips, so nothing uploads and no known flakes are quarantined. TRUNK_QUARANTINE_ENABLED- # splits the two — it defaults off, so set it to 'true' to let the verdict step below mask- # known flakes; otherwise any real failure reds the job (quarantining needs upload on too).- # This inlined gate lacks the shared action's retry.- - name: Quarantine gate (Core)- id: quarantine_gate_core- continue-on-error: true- if: ${{ !cancelled() && matrix.segment == 'Core' && needs.changes.outputs.backend == 'true' && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}- uses: trunk-io/analytics-uploader@385f1ccdf345b4532dc4b6c665dd432b702b8e28 # v2.1.2- env:- TRUNK_HIDE_TEST_COLLECTION_LINKS: 'true'- with:- junit-paths: junit-core.xml,junit-core-retry-failures.xml- org-slug: posthog-inc- test-collection-id: AA6dvTdI # backend-core- # Non-default rows upload under their own variant, so a failure only that mode- # hits does not quarantine the default test with the same id.- variant: ${{ matrix.compat && format('compat-{0}', matrix.clickhouse-server-image) || matrix.person-on-events && (matrix.new-events-schema && 'poe-off-events-json' || 'poe-off') || matrix.new-events-schema && 'events-json' || '' }}- cli-version: '0.15.4'- quarantine: true- previous-step-outcome: ${{ steps.run-core-tests.outcome }}- token: ${{ secrets.TRUNK_API_TOKEN }}- - name: Quarantine gate (Temporal)- id: quarantine_gate_temporal- continue-on-error: true- if: ${{ !cancelled() && matrix.segment == 'Temporal' && needs.changes.outputs.backend == 'true' && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}- uses: trunk-io/analytics-uploader@385f1ccdf345b4532dc4b6c665dd432b702b8e28 # v2.1.2- env:- TRUNK_HIDE_TEST_COLLECTION_LINKS: 'true'- with:- junit-paths: junit-temporal.xml,junit-temporal-retry-failures.xml- org-slug: posthog-inc- test-collection-id: smyQjxZ3 # backend-temporal- # A native-JSON row uploads under its own variant, so a failure in that mode- # does not read as a flake of the legacy test with the same id.- variant: ${{ matrix.new-events-schema && 'events-json' || '' }}- cli-version: '0.15.4'- quarantine: true- previous-step-outcome: ${{ steps.run-temporal-tests.outcome }}- token: ${{ secrets.TRUNK_API_TOKEN }}- # Verdict: red a real failure the gate didn't clear as a quarantined flake. != 'success'- # also covers the skipped gate on fork/Dependabot. Passing shards never red here.- - name: Fail on test failure- if: ${{ !cancelled() && ((steps.run-core-tests.outcome == 'failure' && (vars.TRUNK_QUARANTINE_ENABLED != 'true' || steps.quarantine_gate_core.outcome != 'success')) || (steps.run-temporal-tests.outcome == 'failure' && (vars.TRUNK_QUARANTINE_ENABLED != 'true' || steps.quarantine_gate_temporal.outcome != 'success'))) }}- shell: bash- run: exit 1-- # Aggregate and commit snapshot changes from all matrix jobs-- get_clickhouse_versions:- name: Get ClickHouse versions- needs: [changes]- # Only feeds the test matrices, which master pushes skip.- if: needs.changes.outputs.backend == 'true' && github.event_name != 'push'- runs-on: depot-ubuntu-24.04- timeout-minutes: 5- outputs:- # Oldest supported version as plain string (for comparisons)- oldest_supported_image: ${{ steps.read-versions.outputs.oldest_supported_image }}- # Local dev / hobby image, mirrored from clickhouse-versions.json `local`- local_image: ${{ steps.read-versions.outputs.local_image }}- # Fully expanded compat matrix (version x shard group)- compat_matrix: ${{ steps.read-versions.outputs.compat_matrix }}- steps:- # Merge ref: turbo-tests and django check out the merge ref and mount its config.xml.- # Reading the version from another ref would split an atomic version+config bump.- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- sparse-checkout: .github/clickhouse-versions.json- sparse-checkout-cone-mode: false- - name: Read ClickHouse versions from JSON- id: read-versions- run: |- compat_shards="2"- echo "Using $compat_shards shard(s) per compat version"-- # Oldest supported version for main Django tests (for max compatibility)- oldest_supported=$(jq -r '.oldest_supported' .github/clickhouse-versions.json)- if [ -z "$oldest_supported" ] || [ "$oldest_supported" = "null" ]; then- echo "::error::No oldest_supported version found in .github/clickhouse-versions.json"- exit 1- fi- echo "oldest_supported_image=$oldest_supported" >> $GITHUB_OUTPUT- echo "Oldest supported version for Django tests: $oldest_supported"-- # Local dev / hobby image — the version product (turbo) tests run against.- local_image=$(jq -r '.local' .github/clickhouse-versions.json)- if [ -z "$local_image" ] || [ "$local_image" = "null" ]; then- echo "::error::No local version found in .github/clickhouse-versions.json"- exit 1- fi- echo "local_image=$local_image" >> $GITHUB_OUTPUT- echo "Local dev/hobby image for product tests: $local_image"-- # Read all unique versions so we can derive compat coverage.- all=$(jq -c '[.[]] | unique' .github/clickhouse-versions.json)- if [ "$all" = "[]" ] || [ -z "$all" ]; then- echo "::error::No versions found in .github/clickhouse-versions.json"- exit 1- fi- echo "All CH versions found: $all"-- # Compat coverage is only needed for non-oldest versions.- compat_versions=$(jq -c --arg oldest "$oldest_supported" '[.[]] | unique | map(select(. != $oldest))' .github/clickhouse-versions.json)- echo "Compat versions (excluding oldest): $compat_versions"-- compat_count=$(jq -r 'length' <<< "$compat_versions")- echo "compat_count=$compat_count" >> $GITHUB_OUTPUT-- if [ "$compat_count" -eq 0 ]; then- echo "compat_matrix=[]" >> $GITHUB_OUTPUT- echo "No non-oldest CH versions found — compat tests provide no additional coverage, skipping"- else- compat_matrix=$(jq -cn --argjson versions "$compat_versions" --argjson shards "$compat_shards" '- [- $versions[] as $version- | range(1; $shards + 1) as $group- | {- "clickhouse-server-image": $version,- "concurrency": $shards,- "group": $group- }- ]- ')- echo "compat_matrix=$compat_matrix" >> $GITHUB_OUTPUT- echo "Compat matrix: $compat_matrix"- fi- # Aggregate and commit snapshot changes from all matrix jobs- handle-snapshots:- name: Commit snapshot changes- needs: [changes, detect-snapshot-mode, django, turbo-tests]- # Only in UPDATE mode - CHECK mode verifies snapshots match exactly- # Run even if some matrix jobs failed (to commit snapshots from passing jobs)- if: ${{ !cancelled() && needs.detect-snapshot-mode.outputs.mode == 'update' && needs.changes.outputs.backend == 'true' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}- runs-on: ubuntu-24.04- timeout-minutes: 5- permissions:- contents: write- pull-requests: write- steps:- # Look for patches before the checkout. Most runs have none, and the required- # gate waits for this job.- - name: Download all snapshot patches- id: download-patches- continue-on-error: true- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- pattern: snapshot-patch-*- path: /tmp/snapshot-patches- merge-multiple: true-- - name: Check for patches- id: check-patches- run: |- if [ "${{ steps.download-patches.outcome }}" == "failure" ] || [ ! -d /tmp/snapshot-patches ]; then- echo "has-patches=false" >> $GITHUB_OUTPUT- echo "No snapshot patches found"- exit 0- fi-- if [ -n "$(find /tmp/snapshot-patches -name '*.patch' -type f -size +0c 2>/dev/null)" ]; then- echo "has-patches=true" >> $GITHUB_OUTPUT- echo "Found patches:"- ls -la /tmp/snapshot-patches/- else- echo "has-patches=false" >> $GITHUB_OUTPUT- echo "Patch files empty or missing - no snapshot changes"- fi-- # Use GitHub app token so Actions run after commiting updated snapshots- - name: Get app token- if: steps.check-patches.outputs.has-patches == 'true'- id: app-token- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }}- private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }}-- - name: Checkout- if: steps.check-patches.outputs.has-patches == 'true'- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- ref: ${{ github.event.pull_request.head.sha }}- repository: ${{ github.event.pull_request.head.repo.full_name }}- token: ${{ steps.app-token.outputs.token }}- fetch-depth: 1-- - name: Require snapshot conflict checks- if: steps.check-patches.outputs.has-patches == 'true'- run: |- if ! grep -Fq 'marker: snapshot-conflict-exit' .github/actions/commit-snapshots/action.yml; then- echo "::error::Rebase onto master before committing snapshots. This PR head cannot reject conflicting snapshot patches."- exit 1- fi-- - name: Commit snapshots- if: steps.check-patches.outputs.has-patches == 'true'- uses: ./.github/actions/commit-snapshots- with:- workflow-type: backend- patch-path: /tmp/snapshot-patches- snapshot-path: '.'- commit-message: 'test(backend): update query snapshots'- pr-number: ${{ github.event.pull_request.number }}- repository: ${{ github.repository }}- commit-sha: ${{ github.event.pull_request.head.sha }}- branch-name: ${{ github.event.pull_request.head.ref }}- github-token: ${{ steps.app-token.outputs.token }}-- # Job just to collate the status of the matrix jobs for requiring passing status- # Must depend on handle-snapshots to prevent auto-merge before commits complete- django_tests:- needs:- [- changes,- get_clickhouse_versions,- build_django_matrix,- django,- check-migrations,- check-openapi-types,- turbo-discover,- build-product-test-matrix,- turbo-tests,- detect-snapshot-mode,- handle-snapshots,- repo-checks,- sdk-major-guard,- ]- name: Django Tests Pass- runs-on: ubuntu-24.04- permissions:- contents: read- actions: read # the failure rollup downloads each shard's JUnit artifact- # Headroom for downloading every shard's JUnit for the failure rollup.- timeout-minutes: 8- # Intentional !cancelled(): the gate inspects failed dependencies and fails- # closed on live runs; a superseded run records cancelled, not failure.- # The deterministic-failure disjuncts cover this workflow cancelling its- # own run: without them a real repo or OpenAPI failure would report- # cancelled too, and the failure-rate metric would lose it.- if: >- !cancelled()- || needs.repo-checks.outputs.deterministic_failure == 'true'- || needs.check-openapi-types.outputs.deterministic_failure == 'true'- steps:- # Kept on pull requests so the ruleset entry for this job stays satisfied until the- # Depot gate replaces it there.- - name: Point at the Depot gate- if: github.event_name == 'pull_request'- run: echo "::notice::Depot CI runs the backend tests for pull requests. The required check is 'Backend CI on Depot / Test results'."- - name: Summarize dependency results- if: github.event_name != 'pull_request'- run: |- {- echo "### Backend CI dependency results"- echo "| Dependency | Result |"- echo "| --- | --- |"- echo "| repo-checks | ${{ needs.repo-checks.result }} |"- echo "| sdk-major-guard | ${{ needs.sdk-major-guard.result }} |"- echo "| check-openapi-types | ${{ needs.check-openapi-types.result }} |"- echo "| changes | ${{ needs.changes.result }} |"- echo "| get_clickhouse_versions | ${{ needs.get_clickhouse_versions.result }} |"- echo "| build_django_matrix | ${{ needs.build_django_matrix.result }} |"- echo "| django | ${{ needs.django.result }} |"- echo "| check-migrations | ${{ needs.check-migrations.result }} |"- echo "| turbo-discover | ${{ needs.turbo-discover.result }} |"- echo "| build-product-test-matrix | ${{ needs.build-product-test-matrix.result }} |"- echo "| turbo-tests | ${{ needs.turbo-tests.result }} |"- echo "| handle-snapshots | ${{ needs.handle-snapshots.result }} |"- if [[ "${{ needs.repo-checks.outputs.deterministic_failure }}" == "true" ]]; then- echo ""- echo "**Root cause:** Repo checks failed deterministically. Open the repo checks job logs; downstream cancelled jobs are expected."- fi- if [[ "${{ needs.check-openapi-types.outputs.deterministic_failure }}" == "true" ]]; then- echo ""- echo "**Root cause:** OpenAPI type checks failed deterministically. Open the OpenAPI type check job logs; downstream cancelled jobs are expected."- fi- } | tee -a "$GITHUB_STEP_SUMMARY"-- # Per-test failure rollup: only when a test job actually failed, download that- # shard's JUnit so "Check dependency results" can list the failing tests. Skipped- # on green runs (the common case) to avoid pulling every shard artifact for nothing.- # continue-on-error so a flaky fetch can't fail this required gate.- - name: Download shard JUnit results- if: always() && github.event_name != 'pull_request' && (needs.django.result != 'success' || needs.turbo-tests.result != 'success')- continue-on-error: true- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- pattern: '{junit-results-backend,product-junit-results}-*'- path: /tmp/junit-gate/- # Re-run attempts upload under suffixed names, so superseded attempts must be dropped- # here or the rollup below lists attempt-1 failures the re-run already recovered.- - name: Keep only each shard's latest attempt- if: always() && github.event_name != 'pull_request' && (needs.django.result != 'success' || needs.turbo-tests.result != 'success')- continue-on-error: true- run: |- cd /tmp/junit-gate 2>/dev/null || exit 0- shopt -s nullglob- for d in *-attempt[0-9]*; do- base="${d%-attempt*}"- latest="$(printf '%s\n' "$base"-attempt[0-9]* | sort -V | tail -1)"- rm -rf "$base"- [[ "$d" == "$latest" ]] || rm -rf "$d"- done- - name: Check dependency results- run: |- # Per-test failure rollup, emitted from THIS step because GitHub only- # auto-expands the failed step in the log view. Best-effort via `|| true`- # so a parser hiccup never changes the verdict computed below.- python3 <<'PY' || true- import glob, os, xml.etree.ElementTree as ET-- ROOT = "/tmp/junit-gate"- rows = [] # (shard, nodeid, repo_file, message)-- def clean(s):- # Collapse newlines so PR-authored test names can't inject ::workflow-commands::- return " ".join((s or "").split())-- for path in glob.glob(os.path.join(ROOT, "**", "junit-*.xml"), recursive=True):- artifact = os.path.relpath(path, ROOT).split(os.sep)[0]- base = os.path.basename(path)- # Product JUnit is staged as junit-product-<name>.xml; backend keeps its key.- product = base[len("junit-product-"):-len(".xml")] if base.startswith("junit-product-") else ""- shard = f"products/{product}" if product else artifact.removeprefix("junit-results-backend-")- with open(path, "rb") as fh:- raw = fh.read()- # Refuse DTD/entity payloads (XXE, billion-laughs). pytest JUnit has neither,- # so real failures are never dropped; a crafted file is simply skipped.- if b"<!DOCTYPE" in raw or b"<!ENTITY" in raw:- continue- try:- tree = ET.fromstring(raw)- except ET.ParseError:- continue- for tc in tree.iter("testcase"):- bad = tc.find("failure")- if bad is None:- bad = tc.find("error")- if bad is None:- continue- name = tc.get("name", "")- file_ = tc.get("file", "") or tc.get("classname", "")- repo_file = f"products/{product}/{file_}" if product and file_ else file_- nodeid = clean("::".join(p for p in (repo_file, name) if p))- msg = clean(bad.get("message") or bad.text or "")[:200]- rows.append((clean(shard), nodeid, clean(repo_file), msg))-- # Print only when something failed; on a green gate this stays silent.- if rows:- out = ["### Failing backend & product tests", "", f"{len(rows)} failing test(s):",- "", "| Shard | Test | Message |", "| --- | --- | --- |"]- for shard, nodeid, _f, msg in rows[:100]:- out.append(f"| {shard} | `{nodeid.replace('|', r'\|')}` | {msg.replace('|', r'\|')} |")- if len(rows) > 100:- out += ["", f"…and {len(rows) - 100} more."]- text = "\n".join(out) + "\n"- summary = os.environ.get("GITHUB_STEP_SUMMARY")- if summary:- with open(summary, "a") as fh:- fh.write(text)- print(text)- for shard, nodeid, repo_file, msg in rows[:30]:- title = f"Backend test failed ({shard})"- body = f"{nodeid}: {msg}" if msg else nodeid- loc = f"file={repo_file}," if repo_file else ""- print(f"::error {loc}title={title}::{body}")- PY-- if [[ "${{ needs.repo-checks.outputs.deterministic_failure }}" == "true" ]]; then- echo "::error title=Repo checks failed deterministically::Open the repo checks job for the root failure. Backend CI cancelled downstream jobs to save runners; retrying will not fix this."- fi-- if [[ "${{ needs.check-openapi-types.outputs.deterministic_failure }}" == "true" ]]; then- echo "::error title=OpenAPI type checks failed deterministically::Open the OpenAPI type check job for the root failure. Backend CI cancelled downstream jobs to save runners; retrying will not fix this."- fi-- check_required_result() {- local label="$1"- local result="$2"-- if [[ "$result" != "success" && "$result" != "skipped" ]]; then- echo "::error::$label did not succeed (result: '$result')."- exit 1- fi- }-- check_required_result "Repo checks" "${{ needs.repo-checks.result }}"- check_required_result "SDK major guard" "${{ needs.sdk-major-guard.result }}"- check_required_result "OpenAPI type checks" "${{ needs.check-openapi-types.result }}"- # Upstream jobs whose failure would otherwise only cascade into django- # being *skipped*, which this gate reads as success — a green gate with- # zero backend tests run. Naming them makes the failure hit the gate- # directly; legitimate skips (frontend-only PRs) and plain successes still pass.- check_required_result "Change detection" "${{ needs.changes.result }}"- check_required_result "ClickHouse versions" "${{ needs.get_clickhouse_versions.result }}"- check_required_result "Django matrix build" "${{ needs.build_django_matrix.result }}"- check_required_result "Django test matrix" "${{ needs.django.result }}"- check_required_result "Migration checks" "${{ needs.check-migrations.result }}"- check_required_result "Turbo discover" "${{ needs.turbo-discover.result }}"- check_required_result "Product test matrix build" "${{ needs.build-product-test-matrix.result }}"- check_required_result "Product/Turbo tests" "${{ needs.turbo-tests.result }}"- check_required_result "Snapshot mode detection" "${{ needs.detect-snapshot-mode.result }}"- check_required_result "Snapshot commit job" "${{ needs.handle-snapshots.result }}"-- echo "All backend and product checks passed."-- test-selection-verdict:- needs: [django, turbo-tests, changes, turbo-discover]- name: Test selection verdict- # The verdict scores the selection against the JUnit a run produced, so it needs- # a test job to have run. changes is skipped on no-ci drafts. Both matrices are- # skipped on a draft whose selection was untrusted, which defers to the ready run- # — there is no selection to score and no JUnit to score it against. The decision- # itself is still recorded, by capture-test-selection.- if: >-- !cancelled() && github.event_name == 'pull_request' &&- needs.changes.result != 'skipped' &&- (needs.django.result != 'skipped' || needs.turbo-tests.result != 'skipped')- runs-on: ubuntu-24.04- timeout-minutes: 10- permissions:- contents: read- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- fetch-depth: 1000- filter: blob:none-- - name: Install uv- uses: ./.github/actions/setup-uv- with:- enable-cache: false-- - name: Download JUnit artifacts- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- pattern: junit-results-backend-*- path: /tmp/junit-results/- continue-on-error: true-- # Re-run attempts upload under suffixed names; without this the verdict double-counts- # re-executed shards and resurrects failures the re-run already recovered.- - name: Keep only each shard's latest attempt- continue-on-error: true- run: |- cd /tmp/junit-results 2>/dev/null || exit 0- shopt -s nullglob- for d in *-attempt[0-9]*; do- base="${d%-attempt*}"- latest="$(printf '%s\n' "$base"-attempt[0-9]* | sort -V | tail -1)"- rm -rf "$base"- [[ "$d" == "$latest" ]] || rm -rf "$d"- done-- # Reuse the selection artifact from the turbo-discover job rather than- # re-running the snob script. Falls through if the artifact wasn't- # produced (the selector did not run or failed) — the next step- # regenerates it so we still get a shadow verdict for those PRs.- - name: Download selection artifact- id: download-selection- continue-on-error: true- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- name: select-tests-output- path: /tmp/verdict/-- - name: Download the run's sharding plan snapshot- # The verdict reports selected seconds per segment from .test_durations,- # which is not in the repo; use the union turbo-discover snapshotted.- continue-on-error: true- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- name: test-durations-plan- path: .-- - name: Run test selection and verdict- env:- DIFF_BASE: ${{ format('{0}^1', github.sha) }}- PR_NUMBER: ${{ github.event.pull_request.number }}- PR_SHA: ${{ github.event.pull_request.head.sha }}- run: |- set -euo pipefail- mkdir -p /tmp/verdict-- if [[ ! -s /tmp/verdict/selection.json ]]; then- test "$(git rev-parse HEAD)" = "$GITHUB_SHA"- test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3- test "$(git rev-parse HEAD^2)" = "$PR_SHA"- git rev-parse --verify "$DIFF_BASE^{commit}"-- uv run tools/snob_backend_test_selection_shadow.py \- --base-ref "$DIFF_BASE" \- --pretty \- > /tmp/verdict/selection.json- fi-- uv run tools/test_selection_verdict.py \- /tmp/verdict/selection.json \- /tmp/junit-results/ \- --summary-path "$GITHUB_STEP_SUMMARY" \- --pretty \- > /tmp/verdict/verdict.json-- - name: Upload verdict- if: always()- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- with:- name: test-selection-verdict-pr${{ github.event.pull_request.number }}- path: /tmp/verdict/- retention-days: 90-- calculate-running-time:- name: Calculate running time- needs: [changes, django_tests]- runs-on: ubuntu-24.04- timeout-minutes: 5- # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks or Dependabot (no secrets access)- if: >-- !cancelled() && github.actor != 'dependabot[bot]' && (- (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') ||- (github.event_name != 'pull_request' && github.repository == 'PostHog/posthog'))- && needs.changes.result == 'success'- permissions:- contents: read- actions: read- steps:- - name: Get telemetry app token- id: telemetry-app-token- if: github.run_attempt == '1'- continue-on-error: true- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ vars.GH_APP_TELEMETRY_APP_ID }}- private-key: ${{ secrets.GH_APP_TELEMETRY_PRIVATE_KEY }}- - name: Capture running time to PostHog- if: github.run_attempt == '1'- continue-on-error: true- uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0- with:- posthog-token: ${{ secrets.POSTHOG_API_TOKEN }}- event: 'posthog-ci-running-time'- capture-run-duration: true- capture-job-durations: true- github-token: ${{ steps.telemetry-app-token.outputs.token || github.token }}- status-job: 'Django Tests Pass'- runner: 'depot'- - name: Capture running time to DevEx PostHog- if: github.run_attempt == '1'- continue-on-error: true- uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0- with:- posthog-token: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }}- event: 'posthog-ci-running-time'- capture-run-duration: true- capture-job-durations: true- github-token: ${{ steps.telemetry-app-token.outputs.token || github.token }}- status-job: 'Django Tests Pass'- runner: 'depot'-- capture-test-selection:- name: Capture test selection- needs: [turbo-discover]- runs-on: ubuntu-24.04- timeout-minutes: 5- # An empty mode means this run never selected, so there is no decision to report.- # DevEx project only, mirroring monitor-github-rate-limit; continue-on-error so- # telemetry never reds CI. Skipped on forks/Dependabot, which never have the secret.- if: |- !cancelled()- && github.actor != 'dependabot[bot]'- && github.event.pull_request.head.repo.full_name == 'PostHog/posthog'- && needs.turbo-discover.result == 'success'- && needs.turbo-discover.outputs.mode != ''- steps:- - name: Capture test selection to DevEx PostHog- # Depot CI evaluates github.run_attempt as an empty string in a job-level if,- # so this guard stays on the step for the Depot copy of this job to run.- if: github.run_attempt == '1'- continue-on-error: true- uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0- with:- posthog-token: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }}- event: 'posthog-ci-test-selection'- properties: ${{ needs.turbo-discover.outputs.telemetry }}-- report-test-timings:- name: Report per-test traces- # django and turbo-tests are here for their results only; django_tests already- # waits for both, so this adds no wall time.- needs: [django_tests, django, turbo-tests]- runs-on: ubuntu-24.04- continue-on-error: true- timeout-minutes: 10- # Runs on every attempt: a re-run passing what an earlier attempt failed is the same-commit- # flake proof the test-health queue reads, and the shards a re-run re-executed upload under- # attempt-suffixed artifact names so only they are emitted for the new attempt.- # Manual master runs and all in-repo PRs. Skip forks (no secrets access)- # and master pushes (their matrices are skipped, so there are no traces).- # Both matrices skipped also means no JUnit was uploaded, so there are no- # traces to emit. This job sits on the tail of the run, so skipping it takes- # its whole duration off the wall of a draft that ran no tests.- if: >-- !cancelled() &&- github.repository == 'PostHog/posthog' &&- (needs.django.result != 'skipped' || needs.turbo-tests.result != 'skipped') &&- (- (github.event_name != 'pull_request' && github.event_name != 'push' && github.ref == 'refs/heads/master') ||- (github.event_name == 'pull_request' &&- github.event.pull_request.head.repo.full_name == 'PostHog/posthog')- )- steps:- # Labelled PRs run the trusted base parser, so the same PR can't swap the script that receives secrets.- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}- fetch-depth: 1- - name: Check for trusted timing reporter- id: timing-reporter- shell: bash- run: |- # The owners resolver is a path dep of the reporter, so a base ref predating it- # can't resolve its deps. A base ref from before the move to packages/owners-yaml still- # carries it at tools/owners, so both locations count as available.- available=false- if [[ -f .github/scripts/report_test_timings.py && ( -d packages/owners-yaml || -d tools/owners ) ]]; then- available=true- fi- # A re-run may emit only through an attempt-aware reporter: an older script would- # re-report every attempt-1 shard under the new attempt's resource attributes.- if [[ "$GITHUB_RUN_ATTEMPT" != "1" ]] && ! grep -q "def partition_run_attempt" .github/scripts/report_test_timings.py 2>/dev/null; then- available=false- fi- if [[ "$available" != "true" ]]; then- echo "Trusted timing reporter is not available on the checked-out ref for attempt $GITHUB_RUN_ATTEMPT; skipping emit."- fi- echo "available=$available" >> "$GITHUB_OUTPUT"- # Direct install: The trusted PR base used for reporting may predate the local action.- - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0- if: steps.timing-reporter.outputs.available == 'true'- with:- version: '0.13.4'- - name: Download junit artifacts- if: steps.timing-reporter.outputs.available == 'true'- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- path: ./junit-artifacts- pattern: '{junit-results-backend,product-junit-results}-*'- - name: Emit per-test traces- if: steps.timing-reporter.outputs.available == 'true'- continue-on-error: true- env:- POSTHOG_DEVEX_PROJECT_API_TOKEN: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }}- # dual emission while per-test traces migrate to the project behind POSTHOG_API_TOKEN- POSTHOG_CI_TRACES_EXTRA_TOKEN: ${{ secrets.POSTHOG_API_TOKEN }}- POSTHOG_OTLP_TRACES_ENDPOINT: ${{ vars.POSTHOG_OTLP_TRACES_ENDPOINT || 'https://us.i.posthog.com/i/v1/traces' }}- run: |- uv run --script .github/scripts/report_test_timings.py \- --min-duration-seconds=0.5 \- ./junit-artifacts-- backend-coverage-report:- name: Backend coverage report- needs: [changes, turbo-tests, django_tests, turbo-discover]- runs-on: ubuntu-24.04- continue-on-error: true # report-only, never block a PR- timeout-minutes: 10- # Pinned job-level: this job runs PR-head-controlled scripts with github.token, so its- # grants must never silently widen if the workflow-level permissions block ever does.- permissions:- contents: read- pull-requests: write # sticky CI report comment- # In-repo PRs only — fork PRs get a read-only token that can't comment. Runs when either- # the product (turbo-tests) or core (django_tests) suite ran, so a core-only PR still- # gets a comment; both skip on drafts, leaving nothing to report.- # A narrowed run measures only the selected tests, so patch coverage would report- # gaps that the full suite covers. Report only when the Django matrix ran full.- # Security invariant: this job runs PR-head-controlled scripts (coverage_report.py,- # post-coverage-section.mjs) — never add secrets beyond github.token to this job- # (report-test-timings checks out the base ref for exactly this reason).- if: >-- !cancelled() &&- github.event_name == 'pull_request' &&- github.event.pull_request.draft == false &&- needs.turbo-discover.outputs.mode != 'selected' &&- github.event.pull_request.head.repo.full_name == 'PostHog/posthog' &&- (needs.turbo-tests.result != 'skipped' || needs.django_tests.result != 'skipped')- && needs.changes.result == 'success'- steps:- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- with:- # Default pull_request checkout = the synthetic merge ref — the same tree the- # product coverage runs measured, so diff line numbers match. The coverage- # conversion reads its source files from it. Full history for diff-cover;- # blob:none keeps it light (commit graph only, blobs fetched on demand).- fetch-depth: 0- filter: blob:none- - name: Check for coverage reporter- id: reporter- shell: bash- run: |- if [[ -f .github/scripts/coverage_report.py ]]; then- echo "available=true" >> "$GITHUB_OUTPUT"- else- echo "available=false" >> "$GITHUB_OUTPUT"- echo "Coverage reporter not present on this ref; skipping (branch may predate the check)."- fi- - name: Download coverage artifacts- if: steps.reporter.outputs.available == 'true'- continue-on-error: true # a run where no shard emitted coverage data is not a failure- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- path: ./cov-artifacts- pattern: coverage-products-*- - name: Download core coverage artifacts- if: steps.reporter.outputs.available == 'true'- continue-on-error: true # core coverage only present once the Core segment ran with --cov- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0- with:- path: ./core-artifacts- pattern: coverage-core-*- - uses: ./.github/actions/setup-uv- with:- enable-cache: false- if: steps.reporter.outputs.available == 'true'- - name: Build coverage report- # diff-cover is declared inline (PEP 723) in coverage_report.py; uv provisions it.- if: steps.reporter.outputs.available == 'true'- run: |- mkdir -p cov-artifacts core-artifacts- git fetch --no-tags --filter=blob:none --quiet origin master- uv run --script .github/scripts/coverage_report.py \- --artifacts cov-artifacts \- --core-artifacts core-artifacts \- --combined-out combined-coverage.xml \- --compare-branch origin/master \- --patch-json-out patch-coverage.json \- --out coverage-comment.md- - name: Post to CI report comment- if: steps.reporter.outputs.available == 'true'- env:- GITHUB_TOKEN: ${{ github.token }}- run: |- if [ -f .github/scripts/post-coverage-section.mjs ]; then- node .github/scripts/post-coverage-section.mjs coverage-comment.md patch-coverage.json- fi- - name: Upload patch coverage payload- if: ${{ always() && steps.reporter.outputs.available == 'true' }}- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0- with:- name: patch-coverage- path: |- patch-coverage.json- combined-coverage.xml- coverage-comment.md- if-no-files-found: ignore- retention-days: 2- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwisediff --git a/.github/workflows/ci-backend-shadow-drift.yml b/.github/workflows/ci-depot-action-mirrors.ymlsimilarity index 60%rename from .github/workflows/ci-backend-shadow-drift.ymlrename to .github/workflows/ci-depot-action-mirrors.ymlindex 3cf1a1e0e7cf..30bbbed83c50 100644--- a/.github/workflows/ci-backend-shadow-drift.yml+++ b/.github/workflows/ci-depot-action-mirrors.yml@@ -1,21 +1,12 @@-name: Backend CI shadow drift check--# Fails when the canonical backend workflow or a mirrored local action changes-# without a matching update to the Depot shadow. The workflow is hand-maintained-# (see its header comment for intentional deltas), while the actions are vendored;-# without this check, canonical drifts silently and the Depot timing trial stops-# measuring the same work. The check lives in depot_mirrors.py, which-# `hogli ci:preflight` also runs before a push.-#-# Cheap to run: it triggers only on changes to the backend workflow, local actions, Depot files, or the check itself.-# Always enforced: every pull request's backend tests run on Depot, so the mirror and-# canonical have to agree on every commit.+name: Depot action mirrors+# Depot CI resolves `./.github/actions/<name>` from .depot/actions/<name>, so every local action a+# Depot workflow uses has a mirror there. This check fails when a canonical action changes without+# its mirror, or when an exact mirror drifts. `hogli ci:preflight` runs the same check before a push.on:pull_request:paths:- - .github/workflows/ci-backend-shadow-drift.yml- - .github/workflows/ci-backend.yml+ - .github/workflows/ci-depot-action-mirrors.yml- .github/actions/**- .depot/**- tools/hogli-commands/hogli_commands/depot_mirrors.py@@ -29,7 +20,7 @@ concurrency:jobs:check:- name: Backend CI shadows remain synchronized+ name: Depot action mirrors remain synchronizedruns-on: ubuntu-24.04# 5 min absorbs cold-runner startup + the explicit depth=100 fetch of BASE_SHA below.# `actions/checkout` defaults to depth=1 (HEAD only); a full-history clone is unnecessarydiff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.ymlindex bfda50e42c9b..a4451931c67d 100644--- a/.github/workflows/ci-python.yml+++ b/.github/workflows/ci-python.yml@@ -113,7 +113,6 @@ jobs:# Make sure we run if someone is explicitly change the workflows- .github/workflows/ci-python.yml- .github/workflows/ci-dagster.yml- - .github/workflows/ci-backend.yml- .github/actions/setup-uv/**- .depot/actions/setup-uv/**- '.depot/workflows/*.yml'diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.ymlindex c212856a8277..8a120f1c85c5 100644--- a/.github/workflows/ci-scripts.yml+++ b/.github/workflows/ci-scripts.yml@@ -21,7 +21,6 @@ on:- .oxfmtrc*# turbo-discover.test.js pins the Django segment table to these, and checks# that every path in the events_json list exists.- - .github/workflows/ci-backend.yml- .depot/workflows/ci-backend.yml- .github/new-events-schema-targets.txt# The backend test selector and its verdict script gate ci-backend.diff --git a/posthog/test/repo_invariants/test_events_json_prune_removal.py b/posthog/test/repo_invariants/test_events_json_prune_removal.pyindex 17e20ba641a0..ab0643281fd4 100644--- a/posthog/test/repo_invariants/test_events_json_prune_removal.py+++ b/posthog/test/repo_invariants/test_events_json_prune_removal.py@@ -10,7 +10,6 @@ PRUNE_MODULES = ()PRUNE_WIRING = ("conftest.py",- ".github/workflows/ci-backend.yml",".github/workflows/ci-backend-update-test-timing.yml",".depot/workflows/ci-backend.yml",)diff --git a/posthog/test/repo_invariants/test_signals_scout_tool_citations.py b/posthog/test/repo_invariants/test_signals_scout_tool_citations.pyindex cf6ec2c7e357..a4c1f9c067e3 100644--- a/posthog/test/repo_invariants/test_signals_scout_tool_citations.py+++ b/posthog/test/repo_invariants/test_signals_scout_tool_citations.py@@ -195,8 +195,8 @@ def citation_inputs() -> list[PurePosixPath]:return [PurePosixPath(path.resolve().relative_to(root).as_posix()) for path in paths][email protected]("workflow", [".github/workflows/ci-backend.yml", ".depot/workflows/ci-backend.yml"])-def test_repo_checks_runs_when_a_citation_input_changes(workflow: str) -> None:+def test_repo_checks_runs_when_a_citation_input_changes() -> None:+ workflow = ".depot/workflows/ci-backend.yml"globs = repo_invariants_filter(workflow)# full_match keeps `*` inside one path segment, as picomatch in the paths-filter action does.# fnmatch lets `*` cross `/`, so it would pass a glob that the action never fires on.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.pyindex 4d461d350cd7..8c8a0c0ef0e6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.py@@ -44,7 +44,7 @@ _GENERATED_CONFIG_REFERENCE = re.compile(r"(?:[\w.]+\.)?sources\.generated_confidef _repo_root() -> Path:for parent in Path(__file__).resolve().parents:- if (parent / ".github" / "workflows" / "ci-backend.yml").exists():+ if (parent / ".depot" / "workflows" / "ci-backend.yml").exists():return parentraise RuntimeError("repo root not found")diff --git a/tools/hogli-commands/hogli_commands/ci_preflight.py b/tools/hogli-commands/hogli_commands/ci_preflight.pyindex 6ec3ef0ddcd2..6bef9ba5f080 100644--- a/tools/hogli-commands/hogli_commands/ci_preflight.py+++ b/tools/hogli-commands/hogli_commands/ci_preflight.py@@ -378,9 +378,9 @@ DIFF_CHECKS: list[DiffCheck] = []-# A change under these paths can fail .github/workflows/ci-backend-shadow-drift.yml, which runs+# A change under these paths can fail .github/workflows/ci-depot-action-mirrors.yml, which runs# the same depot_mirrors check. Running it here lands the failure pre-push instead of in CI.-SHADOW_DRIFT_TRIGGERS = [".github/workflows/ci-backend.yml", ".github/actions/**", ".depot/**"]+DEPOT_MIRROR_TRIGGERS = [".github/actions/**", ".depot/**"]def _has_node_modules() -> bool:@@ -864,7 +864,7 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,# A check that can only warn is not worth its run time in the pre-push hook.if chk.matched and not (strict and chk.soft and chk.run is not None):triggered.append(chk)- shadow_drift_triggered = any(matches_globs(path, SHADOW_DRIFT_TRIGGERS) for path in files)+ depot_mirrors_triggered = any(matches_globs(path, DEPOT_MIRROR_TRIGGERS) for path in files)results: list[dict[str, Any]] = []failures = 0@@ -888,15 +888,15 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,click.secho(f" {_ICON[depth_status]} [stack-depth] open layers in this gh stack", fg=_COLOR[depth_status])click.echo(f" {depth_detail}")- if shadow_drift_triggered:+ if depot_mirrors_triggered:violations = mirror_violations(REPO_ROOT, set(files))drift_status: Status = "fail" if violations else "pass"drift_detail = " · ".join(violations[:3]) if violations else "depot mirrors in sync"failures += drift_status == "fail"- results.append({"check": "shadow-drift", "status": drift_status, "files": 1, "detail": drift_detail})+ results.append({"check": "depot-mirrors", "status": drift_status, "files": 1, "detail": drift_detail})if not as_json:click.secho(- f" {_ICON[drift_status]} [shadow-drift] depot shadow drift (.depot mirrors of backend CI)",+ f" {_ICON[drift_status]} [depot-mirrors] depot action mirrors (.depot/actions)",fg=_COLOR[drift_status],)click.echo(f" {drift_detail}")@@ -914,7 +914,7 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,summary = {"changed_files": len(files),- "triggered": (["shadow-drift"] if shadow_drift_triggered else []) + [c.key for c in triggered],+ "triggered": (["depot-mirrors"] if depot_mirrors_triggered else []) + [c.key for c in triggered],"failures": failures,"advisories": advisories,"mode": "fix" if do_fix else ("strict" if strict else "advisory"),@@ -924,7 +924,7 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,if as_json:click.echo(json.dumps(summary))else:- if not triggered and not shadow_drift_triggered:+ if not triggered and not depot_mirrors_triggered:click.secho(" ✓ Nothing in this diff maps to a known CI failure class.", fg="green")click.echo()click.echo(diff --git a/tools/hogli-commands/hogli_commands/depot_mirrors.py b/tools/hogli-commands/hogli_commands/depot_mirrors.pyindex 6ebc1b34a639..bc3200ebc6c2 100644--- a/tools/hogli-commands/hogli_commands/depot_mirrors.py+++ b/tools/hogli-commands/hogli_commands/depot_mirrors.py@@ -1,6 +1,6 @@-"""Keeps the Depot shadow of backend CI in sync with its canonical workflow and local actions.+"""Keeps the Depot CI mirrors of the local actions in sync with their canonical copies.-`.github/workflows/ci-backend-shadow-drift.yml` runs this file with the runner's python3 before any+`.github/workflows/ci-depot-action-mirrors.yml` runs this file with the runner's python3 before anydependency install, and `hogli ci:preflight` imports it, so it must stay standard library only."""@@ -11,8 +11,6 @@ import sysimport subprocessfrom pathlib import Path-CANONICAL_WORKFLOW = ".github/workflows/ci-backend.yml"-SHADOW_WORKFLOW = ".depot/workflows/ci-backend.yml"# These mirrors carry intentional deltas that each mirror's header documents.# Every other mirror stays byte-identical to its canonical action.DELTA_MIRRORS = frozenset({"commit-snapshots", "pnpm-install", "setup-pnpm"})@@ -30,11 +28,6 @@ def _file_bytes(directory: Path) -> dict[Path, bytes]:def mirror_violations(root: Path, changed: set[str]) -> list[str]:violations: list[str] = []- if CANONICAL_WORKFLOW in changed and SHADOW_WORKFLOW not in changed:- violations.append(- f"{CANONICAL_WORKFLOW} changed without a matching update to {SHADOW_WORKFLOW}. "- "Mirror the change, or document it as an intentional delta in the shadow's header."- )for name in sorted(used_mirrors(root)):canonical = f".github/actions/{name}/"mirror = f".depot/actions/{name}/"@@ -54,14 +47,10 @@ def mirror_violations(root: Path, changed: set[str]) -> list[str]:def main(base_sha: str) -> int:diff = subprocess.run(["git", "diff", "--name-only", base_sha, "HEAD"], check=True, capture_output=True, text=True)changed = set(diff.stdout.splitlines())- if SHADOW_WORKFLOW in changed and CANONICAL_WORKFLOW not in changed:- sys.stdout.write(- f"::notice::Only {SHADOW_WORKFLOW} changed. That is allowed, for example to tune a Depot-only knob.\n"- )violations = mirror_violations(Path.cwd(), changed)sys.stdout.writelines(f"::error::{violation}\n" for violation in violations)if not violations:- sys.stdout.write("The Depot shadow is in sync.\n")+ sys.stdout.write("The Depot action mirrors are in sync.\n")return 1 if violations else 0diff --git a/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.py b/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.pyindex 7c19b43d3b9e..e845a84d1051 100644--- a/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.py+++ b/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.py@@ -294,7 +294,7 @@ class TestStalenessRisks:["migrations added on both sides in posthog/migrations"],),(["posthog/api/ours.py"], ["posthog/api/theirs.py"], [], ["master also changed build:openapi"]),- (["a.py"], [".github/workflows/ci-backend.yml"], [], ["CI workflows changed on master (1 file(s))"]),+ (["a.py"], [".github/workflows/ci-python.yml"], [], ["CI workflows changed on master (1 file(s))"]),],)def test_risk_derivation(@@ -396,7 +396,7 @@ class TestWorkspaceScopedLockfile:assert "products/desktop: needs node" in detail-class TestShadowDriftCheck:+class TestDepotMirrorsCheck:@pytest.mark.parametrize("changed,expected_exit,expected_fragment",[@@ -427,4 +427,4 @@ class TestShadowDriftCheck:if expected_fragment:assert expected_fragment in result.outputelse:- assert "shadow-drift" not in result.output+ assert "depot-mirrors" not in result.outputdiff --git a/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.py b/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.pyindex 984e196b0403..a6372e66ff45 100644--- a/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.py+++ b/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.py@@ -58,12 +58,6 @@ class TestMirrorViolations:"but .depot/actions/pnpm-install/ does not exist",id="used-mirror-deleted",),- pytest.param(- {".github/workflows/ci-backend.yml"},- None,- "without a matching update to .depot/workflows/ci-backend.yml",- id="canonical-workflow-only",- ),],)def test_reports_drift(diff --git a/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.py b/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.pyindex 71e2d0966d2e..051dbab4e77c 100644--- a/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.py+++ b/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.py@@ -316,7 +316,7 @@ class TestSemgrepDevex:assert mock_popen.call_args.kwargs["start_new_session"] is True-WORKFLOW = ".github/workflows/ci-backend.yml"+WORKFLOW = ".github/workflows/ci-python.yml"PRODUCT_FILE = "products/x/backend/api.py"EVERY_LANE = ["fe:core", "py:core", "rust:core"]diff --git a/tools/snob_backend_test_selection_shadow.py b/tools/snob_backend_test_selection_shadow.pyindex 4da6fd102a4f..884b64464d85 100755--- a/tools/snob_backend_test_selection_shadow.py+++ b/tools/snob_backend_test_selection_shadow.py@@ -80,7 +80,7 @@ FULL_RUN_PATTERNS = (# see that edge. turbo-discover.js documents this as a live invariant.".test_quarantine.json",# CI / Docker infrastructure- ".github/workflows/ci-backend.yml",+ ".depot/workflows/ci-backend.yml",# A new path in the events_json list has to run on the PR that adds it.".github/new-events-schema-targets.txt",".github/clickhouse-versions.json",diff --git a/tools/workflow-plan/tests/backend-diff.test.ts b/tools/workflow-plan/tests/backend-diff.test.tsindex 749c2ead3e73..75ab3cfd8940 100644--- a/tools/workflow-plan/tests/backend-diff.test.ts+++ b/tools/workflow-plan/tests/backend-diff.test.ts@@ -8,8 +8,7 @@ import { type Context, type JsonValue, evaluateCondition, evaluateTemplate, planimport { type RawStep, type Workflow, loadWorkflow, planWorkflow } from '../src/plan.ts'import { REPO_ROOT, allFiltersChanged, mergeQueue, pullRequest } from '../src/scenarios.ts'-const WORKFLOWS = ['.github/workflows/ci-backend.yml', '.depot/workflows/ci-backend.yml']-const [GITHUB_BACKEND, DEPOT_BACKEND] = WORKFLOWS+const DEPOT_BACKEND = '.depot/workflows/ci-backend.yml'const functions = planFunctions({ dependenciesSucceeded: true, dependenciesFailed: false, cancelled: false })const lowerFile = 'products/engineering_analytics/backend/lower.py'const layerFiles = [@@ -135,10 +134,10 @@ function requiredGate(wf: Workflow, cwd: string, context: Context): SpawnSyncRet}describe('Backend CI comparison boundaries', () => {- it.each(WORKFLOWS)('%s selects a stack layer without counting newer trunk files', (file) => {+ it('selects a stack layer without counting newer trunk files', () => {const repo = createGraph()try {- const wf = loadWorkflow(path.join(REPO_ROOT, file))+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))const context = prContext(repo.merge, repo.head, 'lower')const discovery = stepEnv(wf, step(wf, 'Discover products to test'), context)const selected = repo.git(@@ -182,10 +181,10 @@ describe('Backend CI comparison boundaries', () => {}})- it.each(WORKFLOWS)('%s uses the pinned queue base and disables the PR selector', (file) => {+ it('uses the pinned queue base and disables the PR selector', () => {const repo = createGraph()try {- const wf = loadWorkflow(path.join(REPO_ROOT, file))+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))const context = prContext(repo.queueMerge, repo.merge, 'master', true)const discovery = stepEnv(wf, step(wf, 'Discover products to test'), context)expect(discovery.TURBO_SCM_BASE).toBe(`${repo.queueMerge}^1`)@@ -221,7 +220,7 @@ describe('Backend CI comparison boundaries', () => {(failure) => {const repo = createGraph()try {- const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND!))+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))const context = prContext(repo.merge, failure === 'wrong-head' ? repo.lower : repo.head, 'lower')if (failure === 'wrong-checkout') {repo.git('checkout', '--detach', repo.head)@@ -269,7 +268,7 @@ describe('Backend CI comparison boundaries', () => {])('regenerates a missing verdict selection for $name', ({ queued }) => {const repo = createGraph()try {- const wf = loadWorkflow(path.join(REPO_ROOT, GITHUB_BACKEND!))+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))const sha = queued ? repo.queueMerge : repo.mergeconst context = prContext(sha, queued ? repo.merge : repo.head, queued ? 'master' : 'lower', queued)const target = wf.jobs['test-selection-verdict']!.steps!.find(diff --git a/tools/workflow-plan/tests/workflows.test.ts b/tools/workflow-plan/tests/workflows.test.tsindex d36671ec72f8..afd33d824ff2 100644--- a/tools/workflow-plan/tests/workflows.test.ts+++ b/tools/workflow-plan/tests/workflows.test.ts@@ -88,11 +88,10 @@ const suite = (file: string, selectors: Stubs = {}): ExpectationBuilder => {})}const DEPOT_BACKEND = '.depot/workflows/ci-backend.yml'-const backend = suite('ci-backend.yml', backendSelectors)const depotBackend = suite(DEPOT_BACKEND, backendSelectors)const frontend = suite('ci-frontend.yml', frontendSelectors)const deltalite = suite('build-deltalite.yml')-const PINNED_WORKFLOWS = ['ci-backend.yml', DEPOT_BACKEND, 'ci-frontend.yml']+const PINNED_WORKFLOWS = [DEPOT_BACKEND, 'ci-frontend.yml']interface ReleaseWorkflow {file: string@@ -163,25 +162,6 @@ const allJobsExcept = (file: string, kept: string[]): string[] =>Object.keys(workflow(file).jobs).filter((id) => !kept.includes(id))const EXPECTATIONS: Expectation[] = [- // Depot CI runs every pull request event. GitHub Actions keeps only the `Django Tests Pass`- // job, which points at the Depot gate, so every other job stands down on a pull request.- ...(- [- ['draft PR', pullRequest({ draft: true }), ['dynamic-ci-filter']],- ['ready PR', pullRequest(), ['dynamic-ci-filter']],- ['merge queue', mergeQueue(), []],- ['fork PR', pullRequest({ fork: true }), []],- ['draft PR labeled no-ci', pullRequest({ draft: true, labels: ['no-ci'] }), []],- ] as const- ).map(([name, github, alsoRuns]) =>- backend(- { name, github },- {- runs: ['django_tests', ...alsoRuns],- skipped: allJobsExcept('ci-backend.yml', ['django_tests', ...alsoRuns]),- }- )- ),// The master push runs on Depot CI: the per-commit checks and the schema dump, no matrices.depotBackend({ name: 'master push', github: push() },@@ -208,11 +188,9 @@ const EXPECTATIONS: Expectation[] = [],}),- ...[backend, depotBackend].map((build) =>- build(- { name: 'ready PR, superseded and cancelled', cancelled: true },- { results: { django_tests: 'cancelled' } }- )+ depotBackend(+ { name: 'ready PR, superseded and cancelled', cancelled: true },+ { results: { django_tests: 'cancelled' } }),depotBackend({ name: 'draft PR', github: pullRequest({ draft: true }) },@@ -526,10 +504,22 @@ const STEP_EXPECTATIONS: StepExpectation[] = [scenario: { name: 'hourly schedule', github: schedule() },runs: false,},- // The GitHub Actions backend workflow skips change detection on pull requests, which run on Depot CI.- ...PINNED_WORKFLOWS.filter((file) => file !== 'ci-backend.yml').flatMap((file) => [- { file, job: 'changes', step: 'filter', scenario: { name: 'ready PR', github: pullRequest() }, runs: true },- { file, job: 'changes', step: 'app-token', scenario: { name: 'ready PR', github: pullRequest() }, runs: true },+ ...PINNED_WORKFLOWS.flatMap((file) => [+ {+ file,+ job: 'changes',+ step: 'filter',+ scenario: { name: 'ready PR', github: pullRequest() },+ runs: true,+ },+ { file, job: 'changes', step: 'filter', scenario: { name: 'master push', github: push() }, runs: false },+ {+ file,+ job: 'changes',+ step: 'app-token',+ scenario: { name: 'ready PR', github: pullRequest() },+ runs: true,+ },{file,job: 'changes',@@ -538,13 +528,6 @@ const STEP_EXPECTATIONS: StepExpectation[] = [runs: false,},]),- ...PINNED_WORKFLOWS.map((file) => ({- file,- job: 'changes',- step: 'filter',- scenario: { name: 'master push', github: push() },- runs: false,- })),{ file: 'ci-e2e-playwright.yml', job: 'changes', step: 'schema-key', scenario: E2E_DISPATCH, runs: true },{ file: 'ci-e2e-playwright.yml', job: 'playwright', step: 'schema-cache', scenario: E2E_DISPATCH, runs: true },]@@ -659,7 +642,7 @@ describe('CI workflow run plans', () => {expect(testStep?.run).toMatch(/\bgo test\s+-count=1\b/)})- it('Backend CI runs once every hour, on Depot CI only', () => {+ it('Backend CI runs once every hour', () => {const crons = (file: string): string[] =>((loadWorkflow(path.join(REPO_ROOT, file)).on as { schedule?: { cron: string }[] }).schedule ?? []).map((entry) => entry.cron@@ -669,7 +652,6 @@ describe('CI workflow run plans', () => {'23 */3 * * *','23 1,2,4,5,7,8,10,11,13,14,16,17,19,20,22,23 * * *',])- expect(crons('.github/workflows/ci-backend.yml')).toEqual([])})it.each(['refs/heads/master', ''])('the hourly Depot run keeps master coverage with ref %j', (ref) => {