Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): delete the github backend workflow and router (#116513)

Raúl Negrón-Oteroe1ff77b5da2f
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml
index 8b035741fdda..d29ba2293cd2 100644
--- a/.depot/workflows/ci-backend.yml
+++ b/.depot/workflows/ci-backend.yml
@@ -1,16 +1,11 @@
-# Depot CI copy of .github/workflows/ci-backend.yml. Lives under .depot/ so
-# GitHub Actions ignores it; only Depot CI parses this file. Depot CI runs every
-# pull request event here, and the gate job's own check is the required one.
+# Backend CI. Lives under .depot/ so GitHub Actions ignores it; only Depot CI parses this
+# file. Every pull request event, master push, manual run and the hourly master run start
+# here, and the gate job's `Test results` check is the required one.
#
-# Apples-to-apples is the whole point — keep action pins, env vars, runner
-# sizes, and pytest flags in sync with the canonical workflow. A drift-check
-# job in .github/workflows/ci-backend-shadow-drift.yml fails when canonical
-# changes without a matching depot bump.
-#
-# Intentional deltas vs canonical (everything else is kept apples-to-apples):
-# - Every pull request event runs here. Canonical runs no backend tests on pull requests;
-# its `Django Tests Pass` job only points at this workflow, and it has no push trigger.
-# Depot's own checks carry no output, so the gate posts a `Test results` check with the
+# This file began as a copy of a GitHub Actions workflow, and the local actions it uses are
+# mirrors of .github/actions (.github/workflows/ci-depot-action-mirrors.yml keeps those in
+# sync). The notes below record where it differs from the GitHub Actions shape:
+# - Depot's own checks carry no output, so the gate posts a `Test results` check with the
# failure summary.
# - runs-on: depot-* labels instead of GitHub-hosted ubuntu-24.04
# - turbo-discover adds the `origin` remote before its history fetch when it is missing.
@@ -125,8 +120,6 @@ name: Backend CI on Depot
# Depot CI copy of canonical Backend CI. Branch protection requires the gate job's
# `Backend CI on Depot / Test results` check, posted by the PostHog tests app.
# Pull requests, master pushes, manual runs and the hourly master run all start here.
-# The drift check (.github/workflows/ci-backend-shadow-drift.yml) keeps this file in sync
-# with canonical; it runs whenever either file changes.
on:
push:
branches:
@@ -1374,7 +1367,7 @@ jobs:
fi
overall=0
# A checkout without the matching snapshots must use legacy mode.
- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: product-poe-default" .github/workflows/ci-backend.yml; then
+ if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: product-poe-default" .depot/workflows/ci-backend.yml; then
echo "::notice::Checked-out revision predates the person-on-events default, running product tests in legacy mode"
export PERSON_ON_EVENTS_V2_ENABLED=false
fi
@@ -3578,7 +3571,7 @@ jobs:
COV_ARGS=""
fi
# A checkout without the matching test pins must use legacy mode.
- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: person-on-events-default" .github/workflows/ci-backend.yml; then
+ if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: person-on-events-default" .depot/workflows/ci-backend.yml; then
echo "::notice::Checked-out revision predates the person-on-events default, running Core in legacy mode"
export PERSON_ON_EVENTS_V2_ENABLED=false
fi
diff --git a/.github/scripts/ci_backend_depot_failures.py b/.github/scripts/ci_backend_depot_failures.py
deleted file mode 100644
index 1a75a64a3cf1..000000000000
--- a/.github/scripts/ci_backend_depot_failures.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""Explain why a Depot backend workflow failed: each failed job's step and its log lines.
-
-Depot's own checks carry no output, and its run pages need a Depot login. The GitHub
-check a PR author opens prints these lines instead.
-"""
-
-import re
-import json
-import subprocess
-from typing import Any
-
-RUNNER_MARKER = re.compile(r"^##\[\w+\]")
-# Depot keeps up to 8,000 characters per log line. A shorter cut keeps one long line
-# from filling the annotation.
-MAX_LINE = 500
-
-
-def message(attempt: dict[str, Any]) -> str:
- step = " ".join(f"{attempt['job_display_name']}: {attempt['error_message']}".split())
- lines = (RUNNER_MARKER.sub("", line["content"])[:MAX_LINE] for line in attempt["relevant_lines"])
- return "\n".join((step, *lines))
-
-
-def escape_annotation(text: str) -> str:
- return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
-
-
-def depot_ci(*args: str) -> Any:
- result = subprocess.run(
- ["depot", "ci", *args, "--output", "json"], capture_output=True, text=True, check=True, timeout=60
- )
- return json.loads(result.stdout)
-
-
-def explain(org: str, workflow: str) -> list[str]:
- """Workflow commands: one error annotation per failed step, with its log lines as the message.
-
- The log lines come from pull request code. Escaped as annotation data, the way
- @actions/core escapes it, they stay on the annotation's line and cannot start a command.
- """
- try:
- diagnosis = depot_ci("diagnose", "--workflow", workflow, "--org", org)
- messages = [message(attempt) for attempt in diagnosis["representative_attempts"]]
- except Exception as error:
- return [f"::warning::Cannot read the Depot failure details: {type(error).__name__}"]
- return [f"::error title=Failed on Depot::{escape_annotation(text)}" for text in messages]
diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py
deleted file mode 100644
index 99c864d713b1..000000000000
--- a/.github/scripts/ci_backend_relay.py
+++ /dev/null
@@ -1,575 +0,0 @@
-#!/usr/bin/env python3
-"""Follow the Depot CI run that took over one pull request event, and read one of its checks.
-
-GitHub Actions hands a pull request event to Depot CI (see ci_backend_route.py), and Depot
-starts a workflow for every pull request event. One commit can therefore carry Depot runs of
-earlier events, a duplicate run of the same event, and runs of another pull request with the
-same head. Depot check times and pull request lists cannot tell them apart:
-
-- When Depot cancels a superseded run, it posts a cancelled check for each job that had not
- started, with `started_at` set to the cancel time.
-- Depot's checks sit in its app's check suite for the commit, which carries the branch of the
- commit's first push. When that branch is not the pull request's head, the checks list no
- pull request.
-
-.agents/skills/depot-ci/references/posthog-check-run-semantics.md has the measurements.
-So the Depot wait job's check name carries the event: the pull request number and its
-`updated_at`, which both engines read from the same event payload. That check identifies the
-event's run, and every other check of the run is matched by the Depot workflow id in its
-details URL.
-
-Two events of one commit that arrive within a second or two race the concurrency cancel of
-both engines, and each engine can keep a different event. GitHub Actions alone can also keep
-either event of such a pair, so when this event has no live Depot run, the relay follows the
-run that Depot kept for the racing event.
-
-Standard library only: the relay job runs this with the runner's python3 before any install.
-"""
-
-import os
-import re
-import sys
-import json
-import time
-import subprocess
-import http.client
-import urllib.error
-import urllib.parse
-import urllib.request
-from collections.abc import Callable, Iterable, Sequence
-from dataclasses import dataclass
-from datetime import datetime, timedelta
-from enum import Enum
-from typing import Any, Protocol
-
-from ci_backend_depot_failures import depot_ci, explain
-
-DEPOT_APP_ID = 219785
-DEPOT_ORG = "ntsdt08fpt"
-# The PostHog tests GitHub App. Depot's wait and gate jobs post their verdicts with it too, because
-# Depot posts its own checks from a budget that runs out at peak and then delivers them late.
-MIRROR_APP_ID = 2492437
-# The Trunk merge queue tests each batch through a draft pull request on this branch.
-MERGE_QUEUE_PREFIX = "trunk-merge/"
-
-
-def is_merge_queue(head_ref: str) -> bool:
- return head_ref.startswith(MERGE_QUEUE_PREFIX)
-
-
-DEPOT_WORKFLOW = "Backend CI on Depot"
-WAIT_JOB = "Wait for GitHub Actions to hand off backend tests"
-# Renders the same text as the wait job's name expression in .depot/workflows/ci-backend.yml.
-EVENT_SUFFIX = " (PR {pr}, event {event_at})"
-# How a pull request event payload renders `updated_at`.
-EVENT_TIME = "%Y-%m-%dT%H:%M:%SZ"
-RACING_EVENT_SECONDS = 2
-GATE_CHECK = f"{DEPOT_WORKFLOW} / Django Tests Pass on Depot"
-# The mirror posts the gate under its own name, so that a pull request does not show two rows with one name.
-MIRRORED_GATE_CHECK = f"{DEPOT_WORKFLOW} / Test results"
-# Every name the mirror posts a check under. A Depot run on a workflow revision without the
-# mirror's gate name posts Depot's name from the mirror.
-MIRROR_NAMES = {GATE_CHECK: (GATE_CHECK, MIRRORED_GATE_CHECK)}
-DEPOT_RUN_URL = re.compile(r"^https://depot\.dev/orgs/([^/?]+)/workflows/([a-z0-9]+)(?:[?/]|$)")
-PENDING_STATES = frozenset({"queued", "in_progress", "pending", "waiting", "requested"})
-CONCLUSIONS = frozenset(
- {
- "success",
- "failure",
- "cancelled",
- "skipped",
- "timed_out",
- "neutral",
- "action_required",
- "stale",
- "startup_failure",
- }
-)
-API_ROOT = "https://api.github.com"
-PAGE_SIZE = 100
-# A 403 also covers a secondary rate limit, which clears, so a few refusals in a row are tolerated.
-MAX_REFUSALS = 5
-PREREQUISITES = ("Repo checks (depot-ubuntu-24.04)", "Validate OpenAPI types")
-# Depot's key for the `django_tests` job of .depot/workflows/ci-backend.yml, and how it names job states.
-GATE_JOB_KEY = "ci-backend.yml:django_tests"
-DEPOT_LIVE_STATES = frozenset({"queued", "waiting", "running"})
-DEPOT_JOB_CONCLUSIONS = {"finished": "success", "failed": "failure"}
-# Under the `timeout-minutes` of the `Django Tests Pass` job, so the relay reports before GitHub ends the job.
-GATE_DEADLINE_MINUTES = 90
-
-
-class ReadRefusedError(RuntimeError):
- """The check-runs API keeps refusing the token, so no verdict can be read."""
-
-
-class ReadFailedError(RuntimeError):
- """One read of the check-runs API failed, so this poll cannot tell which checks exist."""
-
-
-@dataclass(frozen=True)
-class CheckRun:
- id: int
- # The conclusion once the check completed, its status before that.
- state: str
- details_url: str
- app_id: int = DEPOT_APP_ID
- # The job attempt in a mirrored check's URL. Depot's own checks carry none.
- attempt: str = ""
-
- @classmethod
- def from_api(cls, run: dict[str, Any]) -> "CheckRun":
- state = (run.get("conclusion") if run.get("status") == "completed" else run.get("status")) or ""
- url = str(run.get("details_url") or "")
- parsed = urllib.parse.urlparse(url)
- query = urllib.parse.parse_qs(parsed.query)
- match = re.fullmatch(rf"/orgs/{re.escape(DEPOT_ORG)}/workflows/([a-z0-9]+)", parsed.path)
- attempt = ""
- if parsed.scheme == "https" and parsed.netloc == "depot.dev" and match:
- job = query.get("job", [""])[0]
- url = f"https://depot.dev/orgs/{DEPOT_ORG}/workflows/{match[1]}"
- if re.fullmatch(r"[a-z0-9]+", job):
- url += f"?job={job}"
- attempt = query.get("attempt", [""])[0]
- else:
- url = ""
- return cls(
- id=int(run["id"]),
- state=state if state in PENDING_STATES | CONCLUSIONS else "unknown",
- details_url=url,
- app_id=int(run["app"]["id"]),
- attempt=attempt if re.fullmatch(r"[a-z0-9]+", attempt) else "",
- )
-
- @property
- def depot_workflow(self) -> str | None:
- match = DEPOT_RUN_URL.match(self.details_url)
- return match.group(2) if match and match.group(1) == DEPOT_ORG else None
-
-
-class Phase(Enum):
- ABSENT = "absent"
- STARTING = "starting"
- DECLINED = "declined"
- CANCELLED = "cancelled"
- RUNNING = "running"
- FINISHED = "finished"
-
-
-@dataclass(frozen=True)
-class Progress:
- phase: Phase
- # The state of the check behind the phase, empty when there is none.
- state: str = ""
- details_url: str = ""
- root_failure: str = ""
- root_check_id: int = 0
-
-
-def event_check_name(job: str, pr_number: int, event_at: str) -> str:
- return f"{DEPOT_WORKFLOW} / {job}{EVENT_SUFFIX.format(pr=pr_number, event_at=event_at)}"
-
-
-def wait_check_name(pr_number: int, event_at: str) -> str:
- return event_check_name(WAIT_JOB, pr_number, event_at)
-
-
-def current_check(checks: Iterable[CheckRun], workflow: str | None) -> CheckRun | None:
- """The check of the newest attempt of one job in `workflow`, from either app that posts it.
-
- Depot posts one check per job attempt but can deliver it late, so its check ids do not order
- its checks against the mirror's. The mirror posts one check per attempt that ran, in order,
- but a post can fail. The mirror decides while it has posted at least as many attempts as
- Depot shows. Otherwise the mirror missed an attempt, and Depot's newest check decides.
- """
- own = [check for check in checks if workflow is not None and check.depot_workflow == workflow]
- mirrored = [check for check in own if check.app_id == MIRROR_APP_ID]
- native = [check for check in own if check.app_id != MIRROR_APP_ID]
- if mirrored and len(native) <= len({check.attempt or str(check.id) for check in mirrored}):
- return max(mirrored, key=lambda check: check.id)
- return max(native, key=lambda check: check.id, default=None)
-
-
-def newest_live(runs: Sequence[CheckRun]) -> CheckRun | None:
- """The newest run that was not cancelled, or the newest cancelled run when every run was."""
- current = [check for workflow in {run.depot_workflow for run in runs} if (check := current_check(runs, workflow))]
- live = [run for run in current if run.state != "cancelled"]
- return max(live or current, key=lambda run: run.id, default=None)
-
-
-def progress(wait: CheckRun | None, checks: Iterable[CheckRun]) -> Progress:
- """Where the Depot run behind `wait` stands, judged by its check among `checks`."""
- if wait is None:
- return Progress(Phase.ABSENT)
- if wait.state == "cancelled":
- return Progress(Phase.CANCELLED, wait.state, wait.details_url)
- if wait.state in PENDING_STATES:
- return Progress(Phase.STARTING, wait.state, wait.details_url)
- if wait.state != "success":
- return Progress(Phase.DECLINED, wait.state, wait.details_url)
- check = current_check(checks, wait.depot_workflow)
- if check is None or check.state in PENDING_STATES:
- return Progress(Phase.RUNNING, check.state if check else "", wait.details_url)
- if check.state == "cancelled":
- return Progress(Phase.CANCELLED, check.state, check.details_url)
- return Progress(Phase.FINISHED, check.state, check.details_url)
-
-
-class CheckReader(Protocol):
- def read(self, name: str) -> list[CheckRun]: ...
-
-
-class CheckRunReader:
- """Reads one commit's Depot check runs by name from each app that posts them, with conditional requests.
-
- A 304 answer is free against the rate limit, so polling stays cheap.
- """
-
- def __init__(
- self,
- repo: str,
- sha: str,
- token: str,
- opener: Callable[..., Any] = urllib.request.urlopen,
- pr_number: int | None = None,
- app_ids: Sequence[int] = (MIRROR_APP_ID, DEPOT_APP_ID),
- ) -> None:
- self._repo = repo
- self._sha = sha
- self._token = token
- self._pr_number = pr_number
- self._opener = opener
- self._app_ids = app_ids
- self._cache: dict[tuple[str, int], tuple[str, list[CheckRun]]] = {}
- self._refusals = 0
-
- def _url(self, name: str, app_id: int, page: int) -> str:
- query = urllib.parse.urlencode(
- {"check_name": name, "app_id": app_id, "filter": "all", "per_page": PAGE_SIZE, "page": page}
- )
- return f"{API_ROOT}/repos/{self._repo}/commits/{self._sha}/check-runs?{query}"
-
- def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]:
- headers = {
- "Accept": "application/vnd.github+json",
- "X-GitHub-Api-Version": "2022-11-28",
- "Authorization": f"Bearer {self._token}",
- }
- if etag:
- headers["If-None-Match"] = etag
- try:
- with self._opener(urllib.request.Request(url, headers=headers), timeout=30) as response:
- raw = response.read(2_000_001)
- if len(raw) > 2_000_000:
- raise ValueError("oversized check response")
- return response.status, response.headers.get("ETag", ""), json.loads(raw.decode("utf-8"))
- except urllib.error.HTTPError as error:
- return error.code, "", {}
-
- def read(self, name: str) -> list[CheckRun]:
- """Every app's checks of `name`, under each name the app posts it with.
-
- `current_check` picks the current one per workflow.
- A failed read of any app raises, because the other app's checks alone can hold a stale attempt.
- """
- runs: list[CheckRun] = []
- for app_id in self._app_ids:
- app_names = MIRROR_NAMES.get(name, (name,)) if app_id == MIRROR_APP_ID else (name,)
- for app_name in app_names:
- app_runs = self._read_app(app_name, app_id)
- if app_runs is None:
- raise ReadFailedError(f"Cannot read {app_name}")
- runs.extend(app_runs)
- return runs
-
- def _read_app(self, name: str, app_id: int) -> list[CheckRun] | None:
- """Read every page, reusing a cached answer only when the API confirms it with 304."""
- etag, cached = self._cache.get((name, app_id), ("", []))
- raw: list[dict[str, Any]] = []
- page = 1
- new_etag = ""
- try:
- while True:
- code, page_etag, body = self._get(self._url(name, app_id, page), etag if page == 1 else "")
- if page == 1 and code == 304:
- self._refusals = 0
- return cached
- if code in (401, 403):
- self._refusals += 1
- if self._refusals >= MAX_REFUSALS:
- raise ReadRefusedError(f"Cannot read checks for {self._sha}")
- if code != 200:
- sys.stdout.write(f"::warning::check-runs API returned {code}\n")
- return None
- self._refusals = 0
- if page == 1:
- new_etag = page_etag
- batch = body["check_runs"]
- if not isinstance(batch, list) or page > 20:
- raise ValueError("invalid or excessive check pages")
- raw.extend(batch)
- if len(batch) < PAGE_SIZE:
- break
- page += 1
- runs = [
- CheckRun.from_api(run)
- for run in raw
- # One malformed record is skipped rather than discarding the whole answer.
- if isinstance(run, dict)
- and (run.get("app") or {}).get("id") == app_id
- and run.get("name") == name
- and run.get("head_sha") == self._sha
- and (
- not run.get("pull_requests")
- or self._pr_number is None
- or any(pr.get("number") == self._pr_number for pr in run["pull_requests"])
- )
- ]
- runs = [run for run in runs if run.depot_workflow is not None]
- except (OSError, http.client.HTTPException, KeyError, TypeError, ValueError, AttributeError):
- sys.stdout.write("::warning::check-runs API read failed\n")
- return None
- # One page's ETag cannot validate the other pages of a paginated response.
- self._cache[(name, app_id)] = (new_etag if page == 1 else "", runs)
- return runs
-
-
-@dataclass(frozen=True)
-class Event:
- repo: str
- sha: str
- pr_number: int
- # The pull request's `updated_at` in this event's payload.
- event_at: str
- merge_queue: bool = False
-
-
-def racing_wait(reader: CheckReader, event: Event, followed: set[str]) -> str | None:
- """The newest racing wait check whose job took the hand-off or is pending, skipping `followed`."""
- event_at = datetime.strptime(event.event_at, EVENT_TIME)
- for offset in range(RACING_EVENT_SECONDS, -RACING_EVENT_SECONDS - 1, -1):
- name = wait_check_name(event.pr_number, (event_at + timedelta(seconds=offset)).strftime(EVENT_TIME))
- if name in followed:
- continue
- wait = newest_live(reader.read(name))
- if progress(wait, ()).phase in (Phase.STARTING, Phase.RUNNING):
- return name
- return None
-
-
-def prerequisite_failure(reader: CheckReader, wait: CheckRun, current: Progress) -> Progress:
- for name in PREREQUISITES:
- latest = current_check(reader.read(f"{DEPOT_WORKFLOW} / {name}"), wait.depot_workflow)
- if latest and latest.state == "failure":
- return Progress(Phase.FINISHED, "failure", current.details_url, name, latest.id)
- return current
-
-
-def poll(
- reader: CheckReader,
- event: Event,
- check_name: str,
- *,
- deadline_minutes: int,
- absent_minutes: int,
- clock: Callable[[], float] = time.monotonic,
- sleep: Callable[[float], None] = time.sleep,
-) -> Progress:
- """Polls until the event's check finishes, Depot declines the hand-off, or a deadline passes.
-
- A duplicate run of the same event can replace a cancelled one, so an absent or cancelled
- run gets `absent_minutes` of grace. After the grace period, the run of a racing event
- stands in for an absent or cancelled one.
- """
- start = clock()
- event_name = wait_check_name(event.pr_number, event.event_at)
- followed = {event_name}
- current = Progress(Phase.ABSENT)
- while True:
- try:
- wait = newest_live(reader.read(event_name))
- checks = reader.read(check_name) if wait and wait.state == "success" else []
- current = progress(wait, checks)
- # Depot cancels its own run only after a deterministic prerequisite failure. A gate that
- # failed without the cancel can follow a retryable one, so it keeps the retry options.
- if current.phase == Phase.CANCELLED and wait and wait.state == "success":
- current = prerequisite_failure(reader, wait, current)
- sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n")
- elapsed = clock() - start
- if current.phase in (Phase.FINISHED, Phase.DECLINED):
- return current
- if current.phase in (Phase.ABSENT, Phase.CANCELLED) and elapsed >= absent_minutes * 60:
- racing = racing_wait(reader, event, followed)
- if racing is None:
- return current
- sys.stdout.write(f"Depot kept a racing event of this commit instead. Following: {racing}\n")
- followed.add(racing)
- event_name = racing
- continue
- except ReadFailedError as error:
- # A failed read says nothing about the run, so it must not end the wait as absent.
- sys.stdout.write(f"::warning::{error}. Reading again.\n")
- elapsed = clock() - start
- if elapsed >= deadline_minutes * 60:
- return current
- # The checked job only posts its check when the matrix is done, so once Depot has
- # started, a slower poll costs at most a minute of latency.
- sleep(60 if current.phase == Phase.RUNNING else 30)
-
-
-def retry_failed_jobs(
- org: str,
- workflow: str,
- *,
- depot: Callable[..., Any] = depot_ci,
- clock: Callable[[], float] = time.monotonic,
- sleep: Callable[[float], None] = time.sleep,
-) -> Progress | None:
- """Retries the workflow's failed jobs on Depot and returns the gate's new verdict, or None when Depot cannot be reached.
-
- Depot refuses a retry while the workflow runs, so the retry waits for the workflow to finish.
- A retry adds one execution to the workflow and queues the gate job in the same call. So the gate's
- state is the new verdict once the workflow has more executions than it had when the retry was sent.
- A gate that another retry turned green in the meantime is relayed as it is.
- """
- start = clock()
- url = f"https://depot.dev/orgs/{org}/workflows/{workflow}"
- executions_at_retry: int | None = None
- failures = 0
- while True:
- sending = False
- try:
- shown = depot("workflow", "show", workflow, "--org", org)
- gate = next(job["status"] for job in shown["jobs"] if job["job_key"] == GATE_JOB_KEY)
- executions = len(shown["executions"])
- status = shown["workflow"]["status"]
- sys.stdout.write(f"Depot workflow: {status}, gate {gate}\n")
- retried = executions_at_retry is not None and executions > executions_at_retry
- if gate == "finished" or (retried and gate not in DEPOT_LIVE_STATES):
- phase = Phase.CANCELLED if gate == "cancelled" else Phase.FINISHED
- return Progress(phase, DEPOT_JOB_CONCLUSIONS.get(gate, gate), url)
- failures = 0
- if executions_at_retry is None and status not in DEPOT_LIVE_STATES:
- sending = True
- depot("retry", shown["run"]["run_id"], "--workflow", workflow, "--org", org, "--failed")
- executions_at_retry = executions
- sys.stdout.write(f"Depot retries the failed jobs: {url}\n")
- except (subprocess.SubprocessError, OSError, ValueError, LookupError, TypeError, StopIteration) as error:
- failures += 1
- sys.stdout.write(f"::warning::Depot CLI call failed: {type(error).__name__}\n")
- # A retry that failed is not sent again, because Depot can have accepted it.
- if sending or failures >= MAX_REFUSALS:
- return None
- if clock() - start >= GATE_DEADLINE_MINUTES * 60:
- return Progress(Phase.RUNNING, details_url=url)
- sleep(30)
-
-
-def gate_verdict(
- reader: CheckReader,
- event: Event,
- *,
- rerun: bool,
- retry: Callable[[str, str], Progress | None] = retry_failed_jobs,
-) -> Progress:
- """The gate verdict that this attempt of the relay job reports.
-
- A GitHub re-run starts nothing on Depot. So when a re-run finds a gate that already failed
- or was cancelled, the relay retries the failed and cancelled Depot jobs and reports the new
- verdict. A cancel is not a test verdict, so the re-run asks for the tests again. A failed
- prerequisite is not retried, because it fails the same way until a commit fixes it.
- """
- if rerun:
- settled = poll(reader, event, GATE_CHECK, deadline_minutes=0, absent_minutes=0)
- target = DEPOT_RUN_URL.match(settled.details_url)
- failed = settled.phase == Phase.FINISHED and settled.state != "success"
- if target and (failed or settled.phase == Phase.CANCELLED) and not settled.root_failure:
- retried = retry(*target.groups())
- if retried:
- return retried
- sys.stdout.write("::warning::Cannot retry on Depot CI. This is the earlier verdict.\n")
- return settled
- return poll(reader, event, GATE_CHECK, deadline_minutes=GATE_DEADLINE_MINUTES, absent_minutes=15)
-
-
-def retry_instructions(event: Event, details_url: str, run_id: str = "") -> list[str]:
- """The ways to run the tests again. A GitHub re-run retries a failed Depot run, so it is listed with a `run_id`."""
- rerun = [
- "Re-run this job on GitHub to retry the failed Depot jobs:",
- f" gh run rerun {run_id} --repo {event.repo} --failed",
- "",
- ]
- lines = [
- f"Backend tests for {event.sha} ran on Depot CI, not GitHub Actions.",
- f"Depot run: {details_url or 'not found'}",
- "",
- *(rerun if run_id else []),
- ]
- # Labels do not route a merge queue batch, and a push to its branch ends the queue attempt.
- if event.merge_queue:
- return [
- *lines,
- "Run on GitHub Actions instead: CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT=0 keeps new merge queue batches there.",
- f" gh variable set CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT --repo {event.repo} --body 0",
- "Then queue the pull request again.",
- ]
- return [
- *lines,
- "Run on GitHub Actions instead: the ci-backend-github label routes the next commit of this PR there.",
- f" gh pr edit {event.pr_number} --repo {event.repo} --add-label ci-backend-github",
- " git commit --allow-empty -m 'chore: retry backend ci on github actions' && git push",
- ]
-
-
-def relay_gate(result: Progress, event: Event, run_id: str) -> tuple[int, list[str]]:
- """The exit code and log lines of the `Django Tests Pass` relay for the gate's progress."""
- if result.phase == Phase.FINISHED and result.state == "success":
- return 0, []
- if result.root_failure:
- return 1, [
- f"::error::{result.root_failure} failed on Depot (check {result.root_check_id}). "
- "Push a fix; a retry will not help."
- ]
- if result.phase == Phase.FINISHED:
- return 1, [
- f"::error::Backend tests on Depot CI concluded {result.state}. This step's log lists the retry options.",
- *retry_instructions(event, result.details_url, run_id),
- ]
- if result.phase == Phase.CANCELLED:
- return 1, [
- f"::error::Depot CI cancelled its run for this event of {event.sha} and started no replacement.",
- *retry_instructions(event, result.details_url, run_id),
- ]
- if result.phase == Phase.DECLINED:
- return 1, [f"::error::Depot declined the hand-off for {event.sha} (wait job: {result.state})"]
- if result.phase == Phase.ABSENT:
- return 1, [f"::error::Depot CI started no run for this event of {event.sha}: no wait job for it"]
- return 1, [f"::error::No Depot verdict for {event.sha} within the relay's deadline"]
-
-
-def main(argv: Sequence[str]) -> int:
- if argv[1:] != ["gate"]:
- sys.stderr.write("usage: ci_backend_relay.py gate\n")
- return 2
- env = os.environ
- event = Event(
- repo=env["REPO"],
- sha=env["SHA"],
- pr_number=int(env["PR_NUMBER"]),
- event_at=env["EVENT_AT"],
- merge_queue=is_merge_queue(env.get("HEAD_REF", "")),
- )
- reader = CheckRunReader(event.repo, event.sha, env["GH_TOKEN"], pr_number=event.pr_number)
- try:
- result = gate_verdict(reader, event, rerun=env.get("GITHUB_RUN_ATTEMPT", "1") != "1")
- except ReadRefusedError as error:
- sys.stdout.write(f"::error::{error}\n")
- return 1
- code, lines = relay_gate(result, event, env.get("GITHUB_RUN_ID", ""))
- if code and result.phase == Phase.FINISHED and (match := DEPOT_RUN_URL.match(result.details_url)):
- lines += explain(*match.groups())
- sys.stdout.writelines(f"{line}\n" for line in lines)
- return code
-
-
-if __name__ == "__main__":
- sys.exit(main(sys.argv))
diff --git a/.github/scripts/ci_backend_route.py b/.github/scripts/ci_backend_route.py
deleted file mode 100644
index 8e89908bd055..000000000000
--- a/.github/scripts/ci_backend_route.py
+++ /dev/null
@@ -1,271 +0,0 @@
-#!/usr/bin/env python3
-"""Decide which CI engine runs a Backend CI event: GitHub Actions or Depot CI.
-
-Only GitHub Actions runs this script. Its answer drives the `Hand off backend tests
-to Depot CI` job, which Depot CI waits for before it runs anything, so Depot never
-routes on its own and the tests never run on both engines. Once that job has concluded
-for a commit, every later run of the same commit repeats its answer, whatever the
-percent or the labels say by then, even after the rollout variable is deleted. A read of
-that record that keeps failing stops routing, so a commit cannot run on both engines.
-
-Any other event goes to Depot only after Depot started a run for it. Depot compiles its run
-from the pull request's merge ref as soon as the event arrives, while GitHub Actions waits
-until GitHub has updated that ref. When the ref is missing, Depot ends the run with no
-workflows; when it stays stale, Depot fails to compile. Depot also cancels some runs under
-its concurrency policy before they start. Each case leaves the event with no Depot run, so
-it stays here.
-
-Merge queue batches have their own percent, CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT, so the
-merge gate moves to Depot separately from source pull requests. Batches ignore the routing
-labels, so setting that percent to 0 keeps every new batch on GitHub Actions.
-"""
-
-import os
-import sys
-import json
-import time
-import hashlib
-import urllib.error
-import urllib.parse
-import urllib.request
-from collections.abc import Callable
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from typing import Any
-
-from ci_backend_relay import (
- EVENT_TIME,
- MIRROR_APP_ID,
- CheckReader,
- CheckRunReader,
- ReadFailedError,
- ReadRefusedError,
- event_check_name,
- is_merge_queue,
-)
-
-LABEL_FORCE_GITHUB = "ci-backend-github"
-LABEL_FORCE_DEPOT = "ci-backend-depot"
-PERCENT_VARIABLE = "CI_BACKEND_DEPOT_PERCENT"
-HANDOFF_CHECK = "Hand off backend tests to Depot CI"
-GITHUB_ACTIONS_APP_ID = 15368
-ENGINE_BY_HANDOFF_CONCLUSION = {"success": "depot", "skipped": "github"}
-API_ROOT = "https://api.github.com"
-API_ATTEMPTS = 3
-API_BACKOFF_SECONDS = 5
-# The first step of Depot's wait job in .depot/workflows/ci-backend.yml posts this check; change both.
-STARTED_JOB = "Depot run started"
-# How long after the event Depot's wait job may take to start before the event stays on GitHub Actions.
-DEPOT_START_SECONDS = 180
-DEPOT_POLL_SECONDS = 10
-
-
-class HandoffReadError(RuntimeError):
- """The durable routing record could not be read, so the event must not be routed."""
-
-
-@dataclass(frozen=True)
-class Decision:
- engine: str
- reason: str
-
-
-def parse_percent(raw: str | None) -> int:
- value = raw.strip() if raw is not None else ""
- # isdecimal() accepts digits int() rejects, and int() refuses very long strings.
- if not value.isascii() or not value.isdecimal():
- return 0
- try:
- return min(int(value), 100)
- except ValueError:
- return 0
-
-
-def bucket_of(pr_number: int) -> int:
- """The pull request's fixed rollout bucket, 0 to 99.
-
- Pull request numbers are sequential, so `pr_number % 100` would give Depot runs of
- consecutive pull requests and then none for the next 95. A hash spreads them while
- every push to one pull request keeps its bucket. Python's `hash()` is salted per
- process, so it would move a pull request between engines from one run to the next.
- """
- return int.from_bytes(hashlib.sha256(str(pr_number).encode()).digest()[:8], "big") % 100
-
-
-def handoff_conclusion(check_runs: list[dict], pr_number: int) -> str | None:
- """The hand-off conclusion that already committed an engine to this commit for this pull request.
-
- A `success` means Depot ran the tests and a `skipped` means GitHub Actions did, and
- neither is undone by a later attempt that was cancelled or re-routed, so the first
- of those found wins over anything newer. GitHub queues a fresh check for every
- attempt, so only concluded ones count, and the API lists every pull request's checks
- for a head SHA, so only this one's count.
- """
- conclusions = [
- run.get("conclusion")
- for run in sorted(check_runs, key=lambda run: run["id"], reverse=True)
- if run.get("status") == "completed"
- and any(pr.get("number") == pr_number for pr in run.get("pull_requests") or [])
- ]
- for committed in ENGINE_BY_HANDOFF_CONCLUSION:
- if committed in conclusions:
- return committed
- return conclusions[0] if conclusions else None
-
-
-def by_bucket(pr_number: int | None, percent: int, label: str) -> Decision:
- if pr_number is None:
- return Decision("github", "no pull request number to hash")
- bucket = bucket_of(pr_number)
- if bucket < percent:
- return Decision("depot", f"{label} {bucket} < {percent}%")
- return Decision("github", f"{label} {bucket} >= {percent}%")
-
-
-def decide(
- event: str,
- percent: int,
- pr_number: int | None,
- labels: list[str],
- is_fork: bool,
- is_draft: bool,
- prior_handoff: str | None = None,
- head_ref: str = "",
- merge_queue_percent: int = 0,
-) -> Decision:
- if event != "pull_request":
- return Decision("github", f"{event} events stay on GitHub Actions")
- prior_engine = ENGINE_BY_HANDOFF_CONCLUSION.get(prior_handoff or "")
- if prior_engine:
- return Decision(prior_engine, f"an earlier run of this commit chose {prior_engine}")
- if is_merge_queue(head_ref):
- return by_bucket(pr_number, merge_queue_percent, "merge queue bucket")
- if LABEL_FORCE_GITHUB in labels:
- return Decision("github", f"label {LABEL_FORCE_GITHUB}")
- if is_fork:
- return Decision("github", "fork pull requests never run on Depot")
- if is_draft and "no-ci" in labels:
- return Decision("github", "no-ci drafts skip on GitHub Actions and run nowhere else")
- if LABEL_FORCE_DEPOT in labels:
- return Decision("depot", f"label {LABEL_FORCE_DEPOT}")
- return by_bucket(pr_number, percent, "bucket")
-
-
-def fetch_handoff_checks(repo: str, sha: str, token: str, *, opener: Any = None) -> list[dict]:
- """GET this commit's hand-off check runs, retrying transient failures only.
-
- 5xx and network errors are worth another attempt; 4xx is not, because with an
- empty rate-limit bucket or a missing permission a retry only spends more of it.
- """
- query = urllib.parse.urlencode(
- {"check_name": HANDOFF_CHECK, "app_id": GITHUB_ACTIONS_APP_ID, "filter": "all", "per_page": 100}
- )
- url = f"{API_ROOT}/repos/{repo}/commits/{sha}/check-runs?{query}"
- request = urllib.request.Request(
- url,
- headers={
- "Accept": "application/vnd.github+json",
- "X-GitHub-Api-Version": "2022-11-28",
- "Authorization": f"Bearer {token}",
- },
- )
- do_open = opener or urllib.request.urlopen
- for attempt in range(1, API_ATTEMPTS + 1):
- try:
- with do_open(request, timeout=30) as response:
- return json.loads(response.read().decode("utf-8"))["check_runs"]
- except urllib.error.HTTPError as error:
- if error.code < 500 or attempt == API_ATTEMPTS:
- raise HandoffReadError(f"GET {url} failed with {error.code}") from error
- except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError, TypeError) as error:
- if attempt == API_ATTEMPTS:
- raise HandoffReadError(f"GET {url} failed: {error}") from error
- sys.stdout.write(f"::warning::Cannot read the earlier hand-off yet ({attempt}/{API_ATTEMPTS})\n")
- time.sleep(API_BACKOFF_SECONDS * attempt)
- raise HandoffReadError(f"GET {url} exhausted {API_ATTEMPTS} attempts")
-
-
-def depot_started(
- reader: CheckReader,
- pr_number: int,
- event_at: str,
- *,
- clock: Callable[[], float] = time.time,
- sleep: Callable[[float], None] = time.sleep,
-) -> bool:
- """Whether Depot started a run for this event, waiting up to DEPOT_START_SECONDS after it.
-
- The started check names the event, so it identifies this event's run and no other.
- """
- try:
- deadline = datetime.strptime(event_at, EVENT_TIME).replace(tzinfo=UTC).timestamp() + DEPOT_START_SECONDS
- except ValueError:
- return False
- name = event_check_name(STARTED_JOB, pr_number, event_at)
- while True:
- try:
- if reader.read(name):
- return True
- except ReadFailedError:
- pass
- except ReadRefusedError:
- return False
- if clock() >= deadline:
- return False
- sleep(DEPOT_POLL_SECONDS)
-
-
-def main() -> int:
- env = os.environ
- event = env.get("EVENT", "")
- pr_raw = env.get("PR_NUMBER", "")
- pr_number = int(pr_raw) if pr_raw.isdigit() else None
- is_fork = env.get("IS_FORK", "false") == "true"
- labels = json.loads(env.get("LABELS") or "null") or []
-
- def route_with(prior_handoff: str | None) -> Decision:
- return decide(
- event=event,
- percent=parse_percent(env.get("PERCENT")),
- pr_number=pr_number,
- labels=labels,
- is_fork=is_fork,
- is_draft=env.get("IS_DRAFT", "false") == "true",
- prior_handoff=prior_handoff,
- head_ref=env.get("HEAD_REF", ""),
- merge_queue_percent=parse_percent(env.get("MERGE_QUEUE_PERCENT")),
- )
-
- prior_handoff = None
- if event == "pull_request" and pr_number is not None and not is_fork:
- try:
- prior_handoff = handoff_conclusion(
- fetch_handoff_checks(env["REPO"], env["SHA"], env["GH_TOKEN"]), pr_number
- )
- except HandoffReadError as error:
- sys.stdout.write(f"::error::Cannot read the earlier hand-off, so this event is not routed: {error}\n")
- return 1
- else:
- sys.stdout.write(f"::notice::Earlier hand-off on this commit: {prior_handoff or 'none'}\n")
- decision = route_with(prior_handoff)
- if (
- decision.engine == "depot"
- and prior_handoff not in ENGINE_BY_HANDOFF_CONCLUSION
- and pr_number is not None
- and not depot_started(
- CheckRunReader(env["REPO"], env["SHA"], env["GH_TOKEN"], pr_number=pr_number, app_ids=(MIRROR_APP_ID,)),
- pr_number,
- env.get("EVENT_AT", ""),
- )
- ):
- decision = Decision("github", "Depot CI started no run for this event")
- sys.stdout.write(f"::notice::Backend CI engine: {decision.engine} ({decision.reason})\n")
- output_path = env.get("GITHUB_OUTPUT")
- if output_path:
- with open(output_path, "a", encoding="utf-8") as handle:
- handle.write(f"engine={decision.engine}\nreason={decision.reason}\n")
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/.github/scripts/ci_flake_overseer.py b/.github/scripts/ci_flake_overseer.py
deleted file mode 100644
index 53a3b045d2d6..000000000000
--- a/.github/scripts/ci_flake_overseer.py
+++ /dev/null
@@ -1,552 +0,0 @@
-#!/usr/bin/env python3
-# ruff: noqa: T201 - CLI output is the contract for this script.
-
-from __future__ import annotations
-
-import os
-import re
-import sys
-import json
-import argparse
-import subprocess
-import urllib.error
-import urllib.request
-from dataclasses import dataclass
-from datetime import datetime
-from pathlib import Path
-from typing import Literal, cast
-
-JsonObject = dict[str, object]
-# Observe-only: we classify failed CI jobs and record what reruns do, but never rerun anything.
-DecisionAction = Literal["observe", "skip deterministic", "skip non-test"]
-
-DEFAULT_ALLOWED_WORKFLOWS = ("Backend CI", "Dagster CI", "E2E CI Playwright")
-DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"
-DECISION_EVENT = "ci_flake_overseer_decision"
-OUTCOME_EVENT = "ci_flake_overseer_rerun_outcome"
-
-
-def compile_patterns(*patterns: str) -> tuple[re.Pattern[str], ...]:
- return tuple(re.compile(pattern, re.IGNORECASE) for pattern in patterns)
-
-
-def compile_classified(*rules: tuple[str, str]) -> tuple[tuple[re.Pattern[str], str], ...]:
- return tuple((re.compile(pattern, re.IGNORECASE), failure_class) for pattern, failure_class in rules)
-
-
-# Jobs/steps whose failures a rerun can't fix — excluded from the flake measurement.
-# Each rule carries a stable failure-class key (matching `hogli ci:preflight` check
-# keys where the two overlap) so dashboards can group deterministic failures that
-# reached CI and join them against local preflight interceptions.
-DETERMINISTIC_JOB_RULES = compile_classified(
- (r"\brepo checks?\b", "repo-checks"),
- (r"\bopenapi\b", "openapi"),
- (r"\bmigrations?\b", "migrations"),
- (r"\blint\b", "lint"),
- (r"\btype ?check\b", "typecheck"),
- (r"\btypescript\b", "typecheck"),
- (r"\bvisual\b", "visual-review"),
- (r"\bsnapshot\b", "snapshot"),
- (r"\bstorybook\b", "storybook"),
- (r"\btach\b", "module-boundaries"),
- (r"\bimport-linter\b", "module-boundaries"),
-)
-
-DETERMINISTIC_STEP_RULES = compile_classified(
- (r"\bcheck module boundaries\b", "module-boundaries"),
- (r"\bproduct facade enforcement\b", "module-boundaries"),
- (r"\bopenapi\b", "openapi"),
- (r"\bmigrations?\b", "migrations"),
- (r"\blint\b", "lint"),
- (r"\btype ?check\b", "typecheck"),
- (r"\btypescript\b", "typecheck"),
- (r"\bvisual review\b", "visual-review"),
- (r"\bsnapshot\b", "snapshot"),
- (r"\bverify changed playwright tests are stable\b", "playwright-stability"),
- (r"\bverify new snapshots\b", "snapshot"),
-)
-
-TEST_STEP_PATTERNS = compile_patterns(
- r"\brun core tests\b",
- r"\brun temporal tests\b",
- r"\brun product tests\b",
- r"\brun dagster tests\b",
- r"\brun playwright tests\b",
-)
-
-TEST_JOB_PATTERNS = compile_patterns(
- r"\bdjango tests\b",
- r"\bproduct tests\b",
- r"\bdagster tests\b",
- r"\bplaywright e2e tests\b",
-)
-
-
-class ExternalCommandError(Exception):
- pass
-
-
-@dataclass(frozen=True)
-class Step:
- name: str
- conclusion: str | None
-
-
-@dataclass(frozen=True)
-class Job:
- id: int
- name: str
- conclusion: str | None
- run_attempt: int
- html_url: str
- # A genuine re-run advances started_at; a carried-over job keeps the prior attempt's value byte-for-byte
- # (run_attempt reports the latest attempt for every job either way), so started_at is the re-run signal.
- started_at: str | None = None
- steps: tuple[Step, ...] = ()
-
- def failed_step_names(self) -> tuple[str, ...]:
- return tuple(step.name for step in self.steps if step.conclusion in {"failure", "timed_out"})
-
-
-@dataclass(frozen=True)
-class WorkflowRun:
- id: int
- workflow_id: int
- name: str
- conclusion: str | None
- head_sha: str
- run_attempt: int
- html_url: str
- head_branch: str = ""
- event: str = ""
- # Empty `pull_requests` (e.g. fork PRs) collapses to None; head_branch stays the primary join key.
- pr_number: int | None = None
-
-
-@dataclass(frozen=True)
-class Decision:
- action: DecisionAction
- reason: str
- job: Job
- # Which signal classified an `observe`: the job name or a failed step. Surfaces silent allowlist drift.
- classified_via: str | None = None
- # Canonical failure-class key for `skip deterministic` decisions (see the rule tables above).
- failure_class: str | None = None
-
-
-def as_str(value: object) -> str | None:
- return value if isinstance(value, str) else None
-
-
-def as_int(value: object) -> int | None:
- if isinstance(value, int):
- return value
- if isinstance(value, str) and value.isdigit():
- return int(value)
- return None
-
-
-def as_object(value: object) -> JsonObject:
- return cast(JsonObject, value) if isinstance(value, dict) else {}
-
-
-def as_object_list(value: object) -> list[JsonObject]:
- if not isinstance(value, list):
- return []
- return [cast(JsonObject, item) for item in value if isinstance(item, dict)]
-
-
-def as_bool_string(value: object) -> bool:
- return str(value).strip().lower() in {"1", "true", "yes", "on"}
-
-
-def run_command(args: tuple[str, ...], timeout_seconds: int = 30) -> str:
- try:
- result = subprocess.run(args, capture_output=True, text=True, timeout=timeout_seconds)
- except (FileNotFoundError, subprocess.TimeoutExpired) as exc:
- raise ExternalCommandError(str(exc)) from exc
- if result.returncode != 0:
- message = (result.stderr or result.stdout).strip() or "command failed"
- raise ExternalCommandError(message)
- return result.stdout
-
-
-def gh_json(repo: str, path: str) -> JsonObject:
- raw = run_command(("gh", "api", f"repos/{repo}/{path}"), timeout_seconds=45)
- parsed = json.loads(raw)
- return parsed if isinstance(parsed, dict) else {}
-
-
-def pr_number_from_object(raw: JsonObject) -> int | None:
- pulls = as_object_list(raw.get("pull_requests"))
- return as_int(pulls[0].get("number")) if pulls else None
-
-
-def workflow_run_from_object(raw: JsonObject) -> WorkflowRun:
- return WorkflowRun(
- id=as_int(raw.get("id")) or 0,
- workflow_id=as_int(raw.get("workflow_id")) or 0,
- name=as_str(raw.get("name")) or as_str(raw.get("workflow_name")) or "",
- conclusion=as_str(raw.get("conclusion")),
- head_sha=as_str(raw.get("head_sha")) or "",
- run_attempt=as_int(raw.get("run_attempt")) or 1,
- html_url=as_str(raw.get("html_url")) or "",
- head_branch=as_str(raw.get("head_branch")) or "",
- event=as_str(raw.get("event")) or "",
- pr_number=pr_number_from_object(raw),
- )
-
-
-def workflow_run_from_event(path: Path) -> WorkflowRun | None:
- try:
- parsed = json.loads(path.read_text())
- except (FileNotFoundError, json.JSONDecodeError):
- return None
- if not isinstance(parsed, dict):
- return None
- workflow_run = as_object(parsed.get("workflow_run"))
- return workflow_run_from_object(workflow_run) if workflow_run else None
-
-
-def job_from_object(raw: JsonObject, run_attempt: int) -> Job:
- return Job(
- id=as_int(raw.get("id")) or 0,
- name=as_str(raw.get("name")) or "",
- conclusion=as_str(raw.get("conclusion")),
- # Trust the job's own attempt over the requested one — the fallback endpoint returns the latest attempt.
- run_attempt=as_int(raw.get("run_attempt")) or run_attempt,
- html_url=as_str(raw.get("html_url")) or "",
- started_at=as_str(raw.get("started_at")),
- steps=tuple(
- Step(name=as_str(step.get("name")) or "", conclusion=as_str(step.get("conclusion")))
- for step in as_object_list(raw.get("steps"))
- ),
- )
-
-
-def fetch_jobs(repo: str, run_id: int, run_attempt: int, *, strict: bool = False) -> tuple[Job, ...]:
- jobs: list[Job] = []
- page = 1
- while True:
- try:
- raw = gh_json(repo, f"actions/runs/{run_id}/attempts/{run_attempt}/jobs?per_page=100&page={page}")
- except ExternalCommandError:
- # The non-attempt endpoint returns the LATEST attempt, so it can't stand in for an arbitrary
- # prior attempt. Strict callers need exact rows, so fail closed (drop any partial pages) rather
- # than return a truncated set or a masquerading attempt.
- if strict:
- return ()
- raw = gh_json(repo, f"actions/runs/{run_id}/jobs?per_page=100&page={page}")
- page_jobs = as_object_list(raw.get("jobs"))
- jobs.extend(job_from_object(job, run_attempt) for job in page_jobs)
- if len(page_jobs) < 100:
- return tuple(jobs)
- page += 1
-
-
-def deterministic_match(job: Job) -> tuple[str, str] | None:
- for pattern, failure_class in DETERMINISTIC_JOB_RULES:
- if pattern.search(job.name):
- return f"job name matches deterministic rule `{pattern.pattern}`", failure_class
- for step_name in job.failed_step_names():
- for pattern, failure_class in DETERMINISTIC_STEP_RULES:
- if pattern.search(step_name):
- return f"failed step `{step_name}` matches deterministic rule `{pattern.pattern}`", failure_class
- return None
-
-
-def test_failure_source(job: Job) -> str | None:
- if any(pattern.search(job.name) for pattern in TEST_JOB_PATTERNS):
- return "job_name"
- if any(pattern.search(step_name) for step_name in job.failed_step_names() for pattern in TEST_STEP_PATTERNS):
- return "step"
- return None
-
-
-def is_test_job_failure(job: Job) -> bool:
- return test_failure_source(job) is not None
-
-
-def classify_job(job: Job) -> Decision:
- deterministic = deterministic_match(job)
- if deterministic is not None:
- reason, failure_class = deterministic
- return Decision(action="skip deterministic", reason=reason, job=job, failure_class=failure_class)
- source = test_failure_source(job)
- if source is None:
- return Decision(action="skip non-test", reason="failed job or step is not an allowlisted test runner", job=job)
- return Decision(
- action="observe", reason="test job failure tracked for rerun outcome", job=job, classified_via=source
- )
-
-
-def classify_failed_jobs(repo: str, run_id: int, run_attempt: int) -> tuple[Decision, ...]:
- return tuple(
- classify_job(job) for job in fetch_jobs(repo, run_id, run_attempt) if job.conclusion in {"failure", "timed_out"}
- )
-
-
-def base_event_properties(repo: str, workflow_run: WorkflowRun) -> JsonObject:
- return {
- "repo": repo,
- "workflow_name": workflow_run.name,
- "run_id": workflow_run.id,
- "run_url": workflow_run.html_url,
- "head_sha": workflow_run.head_sha,
- "head_branch": workflow_run.head_branch,
- "pr_number": workflow_run.pr_number,
- # Master failures block the deploy gate; PR failures only annoy the author — keep them separable.
- "is_master": workflow_run.head_branch in {"master", "main"},
- "trigger_event": workflow_run.event,
- # Reuse the project's existing workflow_run group so events roll up per CI run.
- "$groups": {"workflow_run": str(workflow_run.id)},
- }
-
-
-def build_decision_events(repo: str, workflow_run: WorkflowRun, decisions: tuple[Decision, ...]) -> list[JsonObject]:
- events: list[JsonObject] = []
- for decision in decisions:
- job = decision.job
- properties = base_event_properties(repo, workflow_run)
- properties.update(
- {
- "action": decision.action,
- "reason": decision.reason,
- "classified_via": decision.classified_via,
- "failure_class": decision.failure_class,
- "job_name": job.name,
- "job_id": job.id,
- "job_url": job.html_url,
- "job_conclusion": job.conclusion,
- "failed_steps": list(job.failed_step_names()),
- "run_attempt": job.run_attempt,
- # Dedupe duplicate workflow_run deliveries so the count panels stay accurate.
- "$insert_id": f"decision:{workflow_run.id}:{workflow_run.run_attempt}:{job.id}",
- }
- )
- events.append({"event": DECISION_EVENT, "distinct_id": repo, "properties": properties})
- return events
-
-
-def rerun_outcome_label(conclusion: str | None) -> str:
- if conclusion == "success":
- return "cleared"
- if conclusion in {"failure", "timed_out"}:
- return "still_failing"
- return "unknown"
-
-
-def parse_timestamp(value: str | None) -> datetime | None:
- if not value:
- return None
- try:
- # Normalize a trailing `Z` for parsers that don't accept the military-zulu suffix.
- return datetime.fromisoformat(value.replace("Z", "+00:00"))
- except ValueError:
- return None
-
-
-def job_was_reexecuted(prior_job: Job, current_job: Job) -> bool:
- # A real re-run advances started_at; a carried-over job keeps the prior attempt's value. Compare parsed
- # instants, not raw strings, so a `+00:00` vs `Z` form can't misorder. Missing/unparseable timestamps
- # can't prove re-execution, so treat them as not re-run rather than guess.
- prior = parse_timestamp(prior_job.started_at)
- current = parse_timestamp(current_job.started_at)
- if prior is None or current is None:
- return False
- return current > prior
-
-
-def rerun_outcome(prior_job: Job, current_job: Job | None) -> str:
- # `current_job is None` covers a job missing from the current attempt or an ambiguous (duplicated) name.
- if current_job is None:
- return "unknown"
- if not job_was_reexecuted(prior_job, current_job):
- return "not_rerun"
- return rerun_outcome_label(current_job.conclusion)
-
-
-def index_unique_jobs_by_name(jobs: tuple[Job, ...]) -> dict[str, Job]:
- # Job names are unique within an attempt; if one ever repeats, drop it so an ambiguous match becomes
- # `unknown` instead of silently picking the wrong row.
- by_name: dict[str, Job] = {}
- ambiguous: set[str] = set()
- for job in jobs:
- if job.name in by_name:
- ambiguous.add(job.name)
- by_name[job.name] = job
- for name in ambiguous:
- by_name.pop(name, None)
- return by_name
-
-
-def report_rerun_outcomes(repo: str, workflow_run: WorkflowRun) -> list[JsonObject]:
- # Outcomes only exist once a re-run attempt has completed. Find the test-job failures from the prior
- # attempt, then read how they concluded this attempt — the base-rate signal: do reruns clear flakes?
- if workflow_run.run_attempt <= 1:
- return []
- prior_attempt = workflow_run.run_attempt - 1
- # Fetch the prior attempt strict: the non-attempt fallback returns the LATEST attempt, which would alias
- # as "prior" and collapse the started_at re-execution check. The current attempt IS the latest, so it
- # keeps the fallback (the non-attempt endpoint returns it correctly). Both sides dedupe ambiguous names.
- prior_observed = {
- name: job
- for name, job in index_unique_jobs_by_name(
- fetch_jobs(repo, workflow_run.id, prior_attempt, strict=True)
- ).items()
- if job.conclusion in {"failure", "timed_out"} and classify_job(job).action == "observe"
- }
- if not prior_observed:
- return []
- current_by_name = index_unique_jobs_by_name(fetch_jobs(repo, workflow_run.id, workflow_run.run_attempt))
- events: list[JsonObject] = []
- for job_name in sorted(prior_observed):
- prior_job = prior_observed[job_name]
- current_job = current_by_name.get(job_name)
- properties = base_event_properties(repo, workflow_run)
- properties.update(
- {
- "outcome": rerun_outcome(prior_job, current_job),
- "current_conclusion": current_job.conclusion if current_job else None,
- "prior_conclusion": prior_job.conclusion,
- "job_name": job_name,
- "failed_steps": list(prior_job.failed_step_names()),
- "prior_attempt": prior_attempt,
- "attempt": workflow_run.run_attempt,
- # Dedupe duplicate workflow_run deliveries so the base-rate counts stay accurate.
- "$insert_id": f"outcome:{workflow_run.id}:{workflow_run.run_attempt}:{job_name}",
- }
- )
- events.append({"event": OUTCOME_EVENT, "distinct_id": repo, "properties": properties})
- return events
-
-
-def capture_events(api_key: str, host: str, events: list[JsonObject], timeout_seconds: int = 10) -> None:
- if not api_key or not events:
- return
- payload = json.dumps({"api_key": api_key, "batch": events}).encode()
- request = urllib.request.Request(
- f"{host.rstrip('/')}/batch/",
- data=payload,
- headers={"Content-Type": "application/json"},
- method="POST",
- )
- try:
- # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected -- host is CI config, not request input
- with urllib.request.urlopen(request, timeout=timeout_seconds):
- pass
- except (urllib.error.URLError, TimeoutError, OSError) as exc:
- print(f"::warning title=CI flake overseer telemetry failed::{escape_annotation(str(exc))}")
-
-
-def escape_annotation(value: str) -> str:
- return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
-
-
-def escape_table(value: str) -> str:
- return value.replace("|", "\\|").replace("\n", " ")
-
-
-def markdown_link(label: str, url: str) -> str:
- safe_label = escape_table(label)
- return f"[{safe_label}]({url})" if url else safe_label
-
-
-def format_annotation(decision: Decision) -> str:
- detail = f"{decision.job.name}: {decision.reason}"
- return f"::notice title=CI flake overseer: {decision.action}::{escape_annotation(detail)}"
-
-
-def write_summary(path: Path | None, workflow_run: WorkflowRun, decisions: tuple[Decision, ...]) -> None:
- if path is None:
- return
- lines = [
- "### CI flake overseer (observe-only)",
- "",
- f"- Run: {markdown_link(workflow_run.name or str(workflow_run.id), workflow_run.html_url)}",
- f"- Head SHA: `{workflow_run.head_sha}`",
- f"- Attempt: `{workflow_run.run_attempt}`",
- "",
- "| Job | Decision | Reason |",
- "| --- | --- | --- |",
- ]
- for decision in decisions:
- lines.append(
- f"| {markdown_link(decision.job.name, decision.job.html_url)} | "
- f"`{decision.action}` | {escape_table(decision.reason)} |"
- )
- if not decisions:
- lines.append("| No failed jobs | `skip non-test` | workflow has no failed jobs to inspect |")
- with path.open("a") as summary:
- summary.write("\n".join(lines))
- summary.write("\n")
-
-
-def parse_workflows(value: str | None) -> tuple[str, ...]:
- if not value:
- return DEFAULT_ALLOWED_WORKFLOWS
- workflows = tuple(item.strip() for item in value.split(",") if item.strip())
- return workflows or DEFAULT_ALLOWED_WORKFLOWS
-
-
-def resolve_workflow_run(args: argparse.Namespace) -> WorkflowRun:
- if args.event_path:
- from_event = workflow_run_from_event(Path(args.event_path))
- if from_event is not None and (args.run_id is None or from_event.id == args.run_id):
- return from_event
- if args.run_id is None:
- raise ValueError("--run-id is required when the event payload does not contain workflow_run")
- return workflow_run_from_object(gh_json(args.repo, f"actions/runs/{args.run_id}"))
-
-
-def parse_args(argv: list[str]) -> argparse.Namespace:
- parser = argparse.ArgumentParser(description="Observe failed CI test jobs and record what reruns do.")
- parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "PostHog/posthog"))
- parser.add_argument("--run-id", type=int)
- parser.add_argument("--event-path", default=os.environ.get("GITHUB_EVENT_PATH"))
- parser.add_argument(
- "--enabled", action="store_true", default=as_bool_string(os.environ.get("CI_FLAKE_OVERSEER_ENABLED", ""))
- )
- parser.add_argument("--allowed-workflows", default=os.environ.get("CI_FLAKE_OVERSEER_WORKFLOWS"))
- parser.add_argument("--summary-path", default=os.environ.get("GITHUB_STEP_SUMMARY"))
- # Reuses the DevEx project token already wired into other CI workflows; telemetry no-ops when absent.
- parser.add_argument("--posthog-api-key", default=os.environ.get("POSTHOG_DEVEX_PROJECT_API_TOKEN", ""))
- parser.add_argument("--posthog-host", default=os.environ.get("POSTHOG_DEVEX_PROJECT_HOST") or DEFAULT_POSTHOG_HOST)
- parser.add_argument("--posthog-timeout-seconds", type=int, default=10)
- return parser.parse_args(argv)
-
-
-def main(argv: list[str]) -> int:
- args = parse_args(argv)
- summary_path = Path(args.summary_path) if args.summary_path else None
- try:
- workflow_run = resolve_workflow_run(args)
- except Exception as exc:
- print(f"::warning title=CI flake overseer failed closed::{escape_annotation(str(exc))}")
- return 0
-
- if workflow_run.name not in parse_workflows(args.allowed_workflows):
- print(f"::notice title=CI flake overseer skipped::Workflow `{workflow_run.name}` is not allowlisted")
- return 0
- if not args.enabled:
- print("::notice title=CI flake overseer disabled::Set CI_FLAKE_OVERSEER_ENABLED=true to record observations")
- return 0
-
- # On a re-run attempt, record whether the prior attempt's test failures cleared on rerun.
- events = report_rerun_outcomes(args.repo, workflow_run)
-
- if workflow_run.conclusion in {"failure", "timed_out"}:
- decisions = classify_failed_jobs(args.repo, workflow_run.id, workflow_run.run_attempt)
- for decision in decisions:
- print(format_annotation(decision))
- write_summary(summary_path, workflow_run, decisions)
- events.extend(build_decision_events(args.repo, workflow_run, decisions))
- else:
- print(f"::notice title=CI flake overseer skipped::Workflow conclusion is `{workflow_run.conclusion}`")
-
- capture_events(args.posthog_api_key, args.posthog_host, events, args.posthog_timeout_seconds)
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main(sys.argv[1:]))
diff --git a/.github/scripts/compare-ci-runners.py b/.github/scripts/compare-ci-runners.py
deleted file mode 100755
index 50c89d90c173..000000000000
--- a/.github/scripts/compare-ci-runners.py
+++ /dev/null
@@ -1,526 +0,0 @@
-#!/usr/bin/env python3
-# ruff: noqa: T201 print is the intended output of this CLI
-#
-# THIS SCRIPT IS NO LONGER USED!
-# The Blacksmith trial period is over and we no longer run Blacksmith
-# runners alongside Depot, so there is nothing to pair.
-#
-# Pair Depot and Blacksmith job durations for the 7-day CI trial.
-#
-# Usage:
-# python3 .github/scripts/compare-ci-runners.py --days 7 > report.md
-# python3 .github/scripts/compare-ci-runners.py --since 2026-04-20T18:00:00Z --until 2026-04-21T10:30:00Z > report.md
-# python3 .github/scripts/compare-ci-runners.py --days 7 --format csv > pairs.csv
-#
-# Requires: `gh` CLI authenticated, `jq` not required (we parse JSON in Python).
-# Pulls jobs for listed workflows, groups by (workflow, base_job_name, sha),
-# splits each group into depot-runner vs blacksmith-runner, reports paired deltas.
-#
-# How pairing works:
-# - Matrix-expanded jobs have names like "Python code quality (depot-ubuntu-latest)"
-# and "Python code quality (blacksmith-2vcpu-ubuntu-2404)". Base name stripped
-# by removing trailing " (<runner>)".
-# - Dedicated shadow jobs use the suffix "-blacksmith" or "blacksmith shadow" in name.
-# - Pairs: same (workflow, base_name, sha) → one depot duration + one blacksmith duration.
-#
-# Conclusions:
-# - Speedup uses only success↔success pairs (mixed conclusions would be apples-to-oranges).
-# - Failure rates are tracked per runner across ALL observed runs (paired or not).
-# - Singletons (runs that only appeared on one runner for a given SHA) are reported separately.
-
-import re
-import sys
-import json
-import time
-import argparse
-import statistics
-import subprocess
-from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import UTC, datetime, timedelta
-from pathlib import Path
-
-WORKFLOWS = [
- "ci-backend.yml",
- "ci-dagster.yml",
- "ci-e2e-playwright.yml",
- "ci-mcp.yml",
- "ci-nodejs.yml",
- "ci-proto.yml",
- "ci-python.yml",
- "ci-rust-flags-integration.yml",
- "ci-rust.yml",
- "ci-storybook.yml",
-]
-
-# Matches a runner label at the end of a job name, either alone in trailing parens
-# ("Name (depot-foo)") or as the last comma-separated item ("Name (1/3, depot-foo)").
-# The optional prefix group lets us keep shard/group context so matrix jobs that mix
-# shard and runner in the same parens still pair across providers.
-RUNNER_LABEL_RE = re.compile(
- r"\s*\((?:(?P<prefix>[^()]*?),\s*)?(?P<runner>depot-[a-z0-9.-]+|blacksmith-[a-z0-9.-]+)\s*\)\s*$"
-)
-
-
-DEFAULT_CACHE_DIR = Path(".cache/compare-ci-runners")
-SKIPPED_RUN_CONCLUSIONS = {"skipped", "cancelled", "action_required"}
-
-
-def is_retryable_gh_error(stderr: str) -> bool:
- return "HTTP 5" in stderr or "Server Error" in stderr or "connection reset" in stderr
-
-
-def run_gh(args: list[str], attempts: int = 3) -> str:
- for attempt in range(1, attempts + 1):
- try:
- r = subprocess.run(["gh", *args], capture_output=True, text=True, check=True)
- return r.stdout
- except subprocess.CalledProcessError as e:
- stderr = e.stderr or ""
- if attempt < attempts and is_retryable_gh_error(stderr):
- time.sleep(2 * attempt)
- continue
- if stderr:
- print(stderr.strip(), file=sys.stderr)
- raise
- raise RuntimeError("unreachable")
-
-
-def parse_utc_datetime(value: str) -> datetime:
- try:
- dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
- except ValueError as e:
- raise argparse.ArgumentTypeError(f"invalid datetime: {value}") from e
- if dt.tzinfo is None:
- dt = dt.replace(tzinfo=UTC)
- return dt.astimezone(UTC)
-
-
-def created_dates(since: datetime, until: datetime) -> list[str]:
- day = since.date()
- last_day = until.date()
- dates = []
- while day <= last_day:
- dates.append(day.isoformat())
- day += timedelta(days=1)
- return dates
-
-
-def safe_cache_key(value: str) -> str:
- return re.sub(r"[^A-Za-z0-9_.-]+", "_", value)
-
-
-def load_cached_run_list(cache_dir: Path, workflow: str, created: str) -> list[dict] | None:
- cache_path = cache_dir / "runs" / f"{safe_cache_key(workflow)}-{created}.json"
- try:
- return json.loads(cache_path.read_text())
- except FileNotFoundError:
- return None
- except json.JSONDecodeError:
- return None
-
-
-def save_cached_run_list(cache_dir: Path, workflow: str, created: str, runs: list[dict]) -> None:
- cache_path = cache_dir / "runs" / f"{safe_cache_key(workflow)}-{created}.json"
- cache_path.parent.mkdir(parents=True, exist_ok=True)
- cache_path.write_text(json.dumps(runs, sort_keys=True))
-
-
-def list_runs_for_date(workflow: str, created: str, cache_dir: Path, use_cache: bool) -> tuple[list[dict], bool]:
- if use_cache:
- cached_runs = load_cached_run_list(cache_dir, workflow, created)
- if cached_runs is not None:
- return cached_runs, True
- out = run_gh(
- [
- "run",
- "list",
- "--workflow",
- workflow,
- "--created",
- created,
- "-L",
- "1000",
- "--json",
- "databaseId,headSha,createdAt,conclusion,event",
- ]
- )
- runs = json.loads(out)
- if use_cache:
- save_cached_run_list(cache_dir, workflow, created, runs)
- return runs, False
-
-
-def list_recent_runs(
- workflow: str,
- since: datetime,
- until: datetime,
- include_cancelled: bool,
- cache_dir: Path,
- use_cache: bool,
-) -> tuple[list[dict], dict[str, int], dict[str, int]]:
- # Use GitHub's created filter to avoid walking unrelated pages, then apply
- # exact timestamp bounds locally because --created is day-granular here.
- runs_by_id: dict[int, dict] = {}
- skipped_by_conclusion: dict[str, int] = defaultdict(int)
- cache_stats = {"run_list_cache_hits": 0, "run_list_api_fetches": 0}
- allowed_skipped_conclusions = SKIPPED_RUN_CONCLUSIONS - ({"cancelled"} if include_cancelled else set())
- for created in created_dates(since, until):
- runs, from_cache = list_runs_for_date(workflow, created, cache_dir, use_cache)
- cache_stats["run_list_cache_hits" if from_cache else "run_list_api_fetches"] += 1
- for r in runs:
- run_id = r.get("databaseId")
- if not run_id or run_id in runs_by_id:
- continue
- try:
- ts = datetime.fromisoformat(r["createdAt"].replace("Z", "+00:00"))
- except Exception:
- continue
- if ts < since or ts > until:
- continue
- conclusion = r.get("conclusion") or "unknown"
- if conclusion in allowed_skipped_conclusions:
- skipped_by_conclusion[conclusion] += 1
- continue
- runs_by_id[run_id] = r
- return list(runs_by_id.values()), dict(skipped_by_conclusion), cache_stats
-
-
-def load_cached_jobs(cache_dir: Path, run_id: int) -> list[dict] | None:
- cache_path = cache_dir / "jobs" / f"{run_id}.json"
- try:
- return json.loads(cache_path.read_text())
- except FileNotFoundError:
- return None
- except json.JSONDecodeError:
- return None
-
-
-def save_cached_jobs(cache_dir: Path, run_id: int, jobs: list[dict]) -> None:
- cache_path = cache_dir / "jobs" / f"{run_id}.json"
- cache_path.parent.mkdir(parents=True, exist_ok=True)
- cache_path.write_text(json.dumps(jobs, sort_keys=True))
-
-
-def get_jobs(run_id: int, repo: str, cache_dir: Path, use_cache: bool) -> tuple[list[dict], bool]:
- if use_cache:
- cached_jobs = load_cached_jobs(cache_dir, run_id)
- if cached_jobs is not None:
- return cached_jobs, True
- out = run_gh(
- [
- "api",
- "--paginate",
- "--slurp",
- f"repos/{repo}/actions/runs/{run_id}/jobs",
- ]
- )
- pages = json.loads(out)
- jobs = []
- for page in pages:
- for j in page.get("jobs", []):
- jobs.append(
- {
- "name": j.get("name"),
- "started_at": j.get("started_at"),
- "completed_at": j.get("completed_at"),
- "conclusion": j.get("conclusion"),
- }
- )
- if use_cache:
- save_cached_jobs(cache_dir, run_id, jobs)
- return jobs, False
-
-
-def classify_runner(job_name: str) -> tuple[str, str]:
- """
- Returns (base_name, runner_provider). runner_provider is 'depot', 'blacksmith',
- or 'unknown'. base_name strips the trailing runner label if present, or the
- 'blacksmith shadow' / '-blacksmith' marker.
- """
- # Matrix-expanded form: "Name (depot-foo)", "Name (blacksmith-foo)", or the
- # mixed form "Name (1/3, depot-foo)" where runner shares parens with shard/group.
- m = RUNNER_LABEL_RE.search(job_name)
- if m:
- provider = "depot" if m.group("runner").startswith("depot") else "blacksmith"
- prefix = m.group("prefix")
- head = job_name[: m.start()].rstrip()
- # Preserve non-runner parens content (e.g. shard) so depot and blacksmith
- # variants collapse to the same base name.
- base = f"{head} ({prefix.strip()})" if prefix is not None else head
- return base, provider
-
- # Dedicated shadow form: "... (blacksmith shadow ...)" or "...-blacksmith"
- if "blacksmith shadow" in job_name.lower():
- base = re.sub(r"\s*\(blacksmith shadow[^)]*\)\s*", "", job_name).strip()
- return base, "blacksmith"
- if job_name.endswith("-blacksmith"):
- return job_name[: -len("-blacksmith")], "blacksmith"
-
- # Unrecognized job naming should not be attributed to either provider.
- return job_name, "unknown"
-
-
-def duration_seconds(started: str, completed: str) -> float | None:
- if not started or not completed:
- return None
- try:
- s = datetime.fromisoformat(started.replace("Z", "+00:00"))
- c = datetime.fromisoformat(completed.replace("Z", "+00:00"))
- return (c - s).total_seconds()
- except Exception:
- return None
-
-
-def get_repo() -> str:
- out = run_gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"])
- return out.strip()
-
-
-def collect_jobs_for_runs(
- runs: list[dict], repo: str, cache_dir: Path, use_cache: bool, workers: int
-) -> tuple[list[tuple[dict, list[dict]]], dict[str, int]]:
- stats = {"cache_hits": 0, "api_fetches": 0, "errors": 0}
- if not runs:
- return [], stats
-
- results: list[tuple[dict, list[dict]]] = []
- max_workers = max(1, workers)
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- futures = {executor.submit(get_jobs, run["databaseId"], repo, cache_dir, use_cache): run for run in runs}
- for future in as_completed(futures):
- run = futures[future]
- try:
- jobs, from_cache = future.result()
- except subprocess.CalledProcessError:
- stats["errors"] += 1
- continue
- stats["cache_hits" if from_cache else "api_fetches"] += 1
- results.append((run, jobs))
- return results, stats
-
-
-def derive_stats(by_sha: dict[str, dict[str, dict]]) -> dict:
- """Reduce a (wf, base) sha→provider→{sec,conclusion} map into summary stats."""
- depot_total = depot_fail = 0
- bs_total = bs_fail = 0
- success_pairs: list[tuple[float, float]] = []
- mixed_pairs = 0
- depot_only = bs_only = 0
- for providers in by_sha.values():
- d = providers.get("depot")
- b = providers.get("blacksmith")
- if d:
- depot_total += 1
- if d["conclusion"] == "failure":
- depot_fail += 1
- if b:
- bs_total += 1
- if b["conclusion"] == "failure":
- bs_fail += 1
- if d and b:
- if d["conclusion"] == "success" and b["conclusion"] == "success":
- success_pairs.append((d["sec"], b["sec"]))
- else:
- mixed_pairs += 1
- elif d:
- depot_only += 1
- elif b:
- bs_only += 1
- return {
- "depot_total": depot_total,
- "depot_fail": depot_fail,
- "bs_total": bs_total,
- "bs_fail": bs_fail,
- "success_pairs": success_pairs,
- "mixed_pairs": mixed_pairs,
- "depot_only": depot_only,
- "bs_only": bs_only,
- }
-
-
-def main() -> None:
- ap = argparse.ArgumentParser()
- ap.add_argument("--days", type=int, default=7)
- ap.add_argument("--since", type=parse_utc_datetime)
- ap.add_argument("--until", type=parse_utc_datetime)
- ap.add_argument("--format", choices=["md", "csv"], default="md")
- ap.add_argument("--repo", help="GitHub repo in OWNER/REPO form. Defaults to gh's current repo.")
- ap.add_argument(
- "--workflow",
- action="append",
- dest="workflows",
- help="Workflow file to include. Can be passed multiple times. Defaults to all tracked workflows.",
- )
- ap.add_argument("--workers", type=int, default=8)
- ap.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR)
- ap.add_argument("--no-cache", action="store_true")
- ap.add_argument(
- "--include-cancelled",
- action="store_true",
- help="Fetch jobs for cancelled workflow runs. Useful for forensics, but noisy for runner comparison.",
- )
- args = ap.parse_args()
- if args.until and not args.since:
- ap.error("--until requires --since")
-
- repo = args.repo or get_repo()
- until = args.until or datetime.now(UTC)
- since = args.since or (until - timedelta(days=args.days))
- if since > until:
- ap.error("--since must be before --until")
- use_cache = not args.no_cache
- print(f"# Collecting jobs from {repo} from {since.isoformat()} through {until.isoformat()}", file=sys.stderr)
-
- # observations[(workflow, base_name)][sha][provider] = {"sec": dur, "conclusion": conc}
- observations: dict[tuple[str, str], dict[str, dict[str, dict]]] = defaultdict(lambda: defaultdict(dict))
- total_runs = total_cache_hits = total_api_fetches = total_errors = 0
- total_run_list_cache_hits = total_run_list_api_fetches = 0
- total_skipped_by_conclusion: dict[str, int] = defaultdict(int)
-
- workflows = args.workflows or WORKFLOWS
- for wf in workflows:
- print(f"# {wf}", file=sys.stderr)
- try:
- runs, skipped_by_conclusion, run_list_stats = list_recent_runs(
- wf, since, until, args.include_cancelled, args.cache_dir, use_cache
- )
- except subprocess.CalledProcessError as e:
- print(f" SKIP ({e})", file=sys.stderr)
- continue
- total_run_list_cache_hits += run_list_stats["run_list_cache_hits"]
- total_run_list_api_fetches += run_list_stats["run_list_api_fetches"]
- for conclusion, count in skipped_by_conclusion.items():
- total_skipped_by_conclusion[conclusion] += count
- total_runs += len(runs)
- run_jobs, fetch_stats = collect_jobs_for_runs(runs, repo, args.cache_dir, use_cache, args.workers)
- total_cache_hits += fetch_stats["cache_hits"]
- total_api_fetches += fetch_stats["api_fetches"]
- total_errors += fetch_stats["errors"]
- print(
- f" runs={len(runs)} run_list_cache_hits={run_list_stats['run_list_cache_hits']} "
- f"run_list_api_fetches={run_list_stats['run_list_api_fetches']} "
- f"job_cache_hits={fetch_stats['cache_hits']} job_api_fetches={fetch_stats['api_fetches']} "
- f"errors={fetch_stats['errors']}",
- file=sys.stderr,
- )
- for run, jobs in run_jobs:
- sha = run["headSha"]
- for j in jobs:
- base, provider = classify_runner(j["name"])
- dur = duration_seconds(j["started_at"], j["completed_at"])
- if dur is None:
- continue
- if j["conclusion"] not in ("success", "failure"):
- continue # skipped/cancelled not useful
- observations[(wf, base)][sha][provider] = {
- "sec": dur,
- "conclusion": j["conclusion"],
- }
- skipped_summary = ", ".join(
- f"{conclusion}={count}" for conclusion, count in sorted(total_skipped_by_conclusion.items())
- )
- print(
- "# Summary: "
- f"runs_fetched={total_runs} run_list_cache_hits={total_run_list_cache_hits} "
- f"run_list_api_fetches={total_run_list_api_fetches} job_cache_hits={total_cache_hits} "
- f"job_api_fetches={total_api_fetches} "
- f"errors={total_errors} skipped_before_jobs=({skipped_summary or 'none'})",
- file=sys.stderr,
- )
-
- if args.format == "csv":
- print("workflow,job,sha,depot_sec,blacksmith_sec,depot_conclusion,blacksmith_conclusion,delta_sec,speedup")
- for (wf, base), by_sha in sorted(observations.items()):
- for sha, providers in by_sha.items():
- d = providers.get("depot")
- b = providers.get("blacksmith")
- d_sec = f"{d['sec']:.1f}" if d else ""
- b_sec = f"{b['sec']:.1f}" if b else ""
- d_conc = d["conclusion"] if d else ""
- b_conc = b["conclusion"] if b else ""
- if d and b and d["conclusion"] == "success" and b["conclusion"] == "success" and b["sec"] > 0:
- delta = f"{d['sec'] - b['sec']:.1f}"
- speedup = f"{d['sec'] / b['sec']:.2f}"
- else:
- delta = ""
- speedup = ""
- print(f'"{wf}","{base}",{sha},{d_sec},{b_sec},{d_conc},{b_conc},{delta},{speedup}')
- return
-
- # Markdown summary
- print("# Blacksmith vs Depot — paired CI job durations")
- print(f"\nCollection window: `{since.isoformat()}` through `{until.isoformat()}`. Repo: `{repo}`.\n")
- print(
- "Each row is an `(workflow, job)` with at least one same-SHA success↔success pair. "
- "**Speedup uses success↔success pairs only** — mixed-conclusion pairs are counted in `Mixed` but excluded from the ratio. "
- "`Fails` is the per-runner failure count across all observed runs (paired or not). Durations are seconds.\n"
- )
- print(
- "| Workflow | Job | Success pairs | Depot median | Blacksmith median | Speedup (median) | Depot p95 | Blacksmith p95 | Depot fails | Blacksmith fails | Mixed |"
- )
- print("| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |")
- summary_rows = []
- stats_by_key: dict[tuple[str, str], dict] = {}
- for (wf, base), by_sha in sorted(observations.items()):
- s = derive_stats(by_sha)
- stats_by_key[(wf, base)] = s
- if not s["success_pairs"]:
- continue # no comparable data — surfaced in singletons table below
- depot = [p[0] for p in s["success_pairs"]]
- bs = [p[1] for p in s["success_pairs"]]
- dm = statistics.median(depot)
- bm = statistics.median(bs)
- # Renamed from `speedup` to avoid shadowing the str-typed `speedup` used
- # in the CSV loop above — mypy unifies the type of repeated names in
- # the same function scope and rejects the float here as `Any | float`
- # vs `str`.
- speedup_factor = dm / bm if bm > 0 else float("inf")
- dp95 = sorted(depot)[max(0, int(len(depot) * 0.95) - 1)]
- bp95 = sorted(bs)[max(0, int(len(bs) * 0.95) - 1)]
- dfail = f"{s['depot_fail']}/{s['depot_total']}"
- bfail = f"{s['bs_fail']}/{s['bs_total']}"
- summary_rows.append(
- (speedup_factor, wf, base, len(s["success_pairs"]), dm, bm, dp95, bp95, dfail, bfail, s["mixed_pairs"])
- )
-
- # Sort by speedup descending — biggest wins at the top
- for speedup_factor, wf, base, n, dm, bm, dp95, bp95, dfail, bfail, mixed in sorted(summary_rows, reverse=True):
- print(
- f"| `{wf}` | {base} | {n} | {dm:.0f}s | {bm:.0f}s | **{speedup_factor:.2f}x** | "
- f"{dp95:.0f}s | {bp95:.0f}s | {dfail} | {bfail} | {mixed} |"
- )
-
- # Unpaired / singleton jobs — highlights where the shadow didn't fire consistently
- unpaired_rows = [
- (wf, base, s["depot_only"], s["bs_only"])
- for (wf, base), s in sorted(stats_by_key.items())
- if s["depot_only"] or s["bs_only"]
- ]
- if unpaired_rows:
- print("\n## Unpaired runs")
- print(
- "Jobs that ran on only one runner for a given SHA (shadow didn't fire, paths filter "
- "differed, job was skipped on the other side, etc.). Large imbalances mean the "
- "pairing denominator above is small.\n"
- )
- print("| Workflow | Job | Depot-only | Blacksmith-only |")
- print("| --- | --- | ---: | ---: |")
- for wf, base, d_only, b_only in unpaired_rows:
- print(f"| `{wf}` | {base} | {d_only} | {b_only} |")
-
- print("\n## Interpretation")
- print(
- "- **Speedup > 1.0** → Blacksmith is faster on that job\n"
- "- **Speedup < 1.0** → Blacksmith is slower (decision: is the price cut worth it?)\n"
- "- **Fails (x/y)** → per-runner failure count across all observed runs. High asymmetry "
- "(e.g. `5/50` vs `1/50`) points to runner-specific instability\n"
- "- **Mixed** → pairs where one runner passed and the other failed. Excluded from speedup "
- "but worth inspecting manually — the shadow may be masking a flake or a real break\n"
- "- Low `Success pairs` count (< 10) → treat as noisy; widen window or re-run\n"
- "- Compare `p95` to `median` — a large gap means runner variance is high\n"
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/.github/scripts/depot_backend_runs.py b/.github/scripts/depot_backend_runs.py
index 497ce75d0a39..3840c0909f3a 100644
--- a/.github/scripts/depot_backend_runs.py
+++ b/.github/scripts/depot_backend_runs.py
@@ -36,7 +36,6 @@ from pathlib import Path
from typing import TypedDict, cast
REPO = "PostHog/posthog"
-# Both match .github/scripts/ci_backend_relay.py, which reads the same workflow for pull requests.
WORKFLOW_NAME = "Backend CI on Depot"
GATE_JOB_KEY = "ci-backend.yml:django_tests"
GATE_CONCLUSIONS = {"finished": "success", "failed": "failure", "cancelled": "cancelled"}
diff --git a/.github/scripts/test_ci_backend_depot_failures.py b/.github/scripts/test_ci_backend_depot_failures.py
deleted file mode 100644
index 36b4c12d1595..000000000000
--- a/.github/scripts/test_ci_backend_depot_failures.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import json
-import subprocess
-from typing import Any
-
-import pytest
-
-import ci_backend_depot_failures as depot_failures
-
-
-def stub_diagnose(monkeypatch: pytest.MonkeyPatch, outcome: str | BaseException) -> None:
- def run(*args: Any, **kwargs: Any) -> subprocess.CompletedProcess[str]:
- if isinstance(outcome, BaseException):
- raise outcome
- return subprocess.CompletedProcess(args, 0, stdout=outcome)
-
- monkeypatch.setattr(depot_failures.subprocess, "run", run)
-
-
-def test_log_lines_stay_inside_one_escaped_annotation(monkeypatch: pytest.MonkeyPatch) -> None:
- lines = ["##[error]FAILED test_widget - 100% broken", "::add-mask::ghp_example", "tail\r\n::set-output name=x::y"]
- attempt = {
- "job_display_name": "Repo checks",
- "error_message": "Step 3 (Validate widgets): exit 1",
- "relevant_lines": [{"content": line} for line in lines],
- }
- stub_diagnose(monkeypatch, json.dumps({"representative_attempts": [attempt]}))
-
- assert depot_failures.explain("org", "workflow") == [
- "::error title=Failed on Depot::Repo checks: Step 3 (Validate widgets): exit 1"
- "%0AFAILED test_widget - 100%25 broken"
- "%0A::add-mask::ghp_example"
- "%0Atail%0D%0A::set-output name=x::y"
- ]
-
-
- "outcome, error",
- [
- (FileNotFoundError("depot"), "FileNotFoundError"),
- (subprocess.CalledProcessError(1, "depot"), "CalledProcessError"),
- ("not json", "JSONDecodeError"),
- ],
-)
-def test_unreadable_diagnosis_leaves_one_warning(
- monkeypatch: pytest.MonkeyPatch, outcome: str | BaseException, error: str
-) -> None:
- stub_diagnose(monkeypatch, outcome)
-
- assert depot_failures.explain("org", "workflow") == [f"::warning::Cannot read the Depot failure details: {error}"]
diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py
deleted file mode 100644
index 39d0c73185c4..000000000000
--- a/.github/scripts/test_ci_backend_relay.py
+++ /dev/null
@@ -1,769 +0,0 @@
-import json
-import subprocess
-import dataclasses
-import http.client
-import urllib.error
-import urllib.parse
-import importlib.util
-import urllib.request
-from collections.abc import Callable, Mapping, Sequence
-from pathlib import Path
-from typing import Any
-
-import pytest
-
-SCRIPT_PATH = Path(__file__).with_name("ci_backend_relay.py")
-SPEC = importlib.util.spec_from_file_location("ci_backend_relay", SCRIPT_PATH)
-assert SPEC is not None
-assert SPEC.loader is not None
-relay = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(relay)
-
-PR = 105723
-EVENT_AT = "2026-09-24T09:54:20Z"
-EVENT = relay.Event(repo="PostHog/posthog", sha="a8a3755cf964", pr_number=PR, event_at=EVENT_AT)
-EVENT_WAIT = relay.wait_check_name(PR, EVENT_AT)
-PLAIN_WAIT = f"{relay.DEPOT_WORKFLOW} / {relay.WAIT_JOB}"
-OLDER_RACING_WAIT = relay.wait_check_name(PR, "2026-09-24T09:54:19Z")
-NEWER_RACING_WAIT = relay.wait_check_name(PR, "2026-09-24T09:54:22Z")
-
-
-def api_run(id: int, state: str, workflow: str = "live", attempt: str = "") -> dict[str, Any]:
- completed = state not in relay.PENDING_STATES
- query = f"job=j1&attempt={attempt}" if attempt else "job=j1&repo=PostHog%2Fposthog"
- return {
- "id": id,
- "status": "completed" if completed else state,
- "conclusion": state if completed else None,
- "app": {"id": relay.MIRROR_APP_ID if attempt else relay.DEPOT_APP_ID},
- "details_url": f"https://depot.dev/orgs/ntsdt08fpt/workflows/{workflow}?{query}",
- }
-
-
-def run(id: int, state: str, workflow: str = "live", started_at: str = "2026-09-24T09:56:00Z") -> Any:
- return relay.CheckRun.from_api({**api_run(id, state, workflow), "started_at": started_at})
-
-
- "event_waits,gates,expected",
- [
- pytest.param([], [], (relay.Phase.ABSENT, ""), id="no run yet"),
- pytest.param([run(1, "in_progress")], [], (relay.Phase.STARTING, "in_progress"), id="waiting for hand-off"),
- pytest.param([run(1, "failure")], [], (relay.Phase.DECLINED, "failure"), id="depot declined"),
- pytest.param([run(1, "cancelled")], [], (relay.Phase.CANCELLED, "cancelled"), id="run cancelled"),
- pytest.param(
- [run(1, "success")],
- [run(9, "cancelled", workflow="stale", started_at="2026-09-24T09:55:14Z")],
- (relay.Phase.RUNNING, ""),
- id="superseded run's cancelled gate is ignored",
- ),
- pytest.param(
- [run(1, "success")],
- [run(9, "cancelled", workflow="stale"), run(10, "success")],
- (relay.Phase.FINISHED, "success"),
- id="this run's gate wins over a newer id elsewhere",
- ),
- pytest.param(
- [run(1, "cancelled", workflow="dup1"), run(2, "success", workflow="dup2")],
- [run(10, "failure", workflow="dup2")],
- (relay.Phase.FINISHED, "failure"),
- id="duplicate run of the event replaces a cancelled one",
- ),
- pytest.param(
- [run(1, "success")],
- [run(10, "cancelled")],
- (relay.Phase.CANCELLED, "cancelled"),
- id="this run cancelled after the hand-off",
- ),
- pytest.param(
- [run(1, "success")],
- [run(10, "failure"), run(11, "success")],
- (relay.Phase.FINISHED, "success"),
- id="retried gate",
- ),
- ],
-)
-def test_progress_of_this_events_run(event_waits: list[Any], gates: list[Any], expected: tuple[Any, str]) -> None:
- wait = relay.newest_live(event_waits)
- result = relay.progress(wait, gates)
- assert (result.phase, result.state) == expected
-
-
-class FakeReader:
- def __init__(self, polls: Sequence[dict[str, list[Any]]], advance_on: str = EVENT_WAIT) -> None:
- self._polls = list(polls)
- self._advance_on = advance_on
- self.poll = 0
- self.reads: list[str] = []
-
- def read(self, name: str) -> list[Any]:
- self.reads.append(name)
- if name == self._advance_on:
- self.poll += 1
- answer = self._polls[min(self.poll, len(self._polls)) - 1].get(name, [])
- if isinstance(answer, Exception):
- raise answer
- return answer
-
-
-class FakeClock:
- def __init__(self) -> None:
- self.now = 0.0
-
- def __call__(self) -> float:
- return self.now
-
- def sleep(self, seconds: float) -> None:
- self.now += seconds
-
-
- "polls,expected,min_minutes",
- [
- pytest.param(
- [{EVENT_WAIT: [run(1, "cancelled", workflow="dup1")]}] * 3
- + [{EVENT_WAIT: [run(1, "cancelled", workflow="dup1"), run(2, "success", workflow="dup2")]}]
- + [
- {
- EVENT_WAIT: [run(1, "cancelled", workflow="dup1"), run(2, "success", workflow="dup2")],
- relay.GATE_CHECK: [run(10, "success", workflow="dup2")],
- }
- ],
- (relay.Phase.FINISHED, "success"),
- 0,
- id="a cancelled run is replaced within the grace window",
- ),
- pytest.param(
- [{EVENT_WAIT: [run(1, "cancelled")]}],
- (relay.Phase.CANCELLED, "cancelled"),
- 15,
- id="a cancelled run fails only after the grace window",
- ),
- pytest.param([{}], (relay.Phase.ABSENT, ""), 15, id="no run fails after the grace window"),
- pytest.param(
- [{EVENT_WAIT: [run(1, "success")]}] * 16
- + [{EVENT_WAIT: relay.ReadFailedError("Cannot read")}]
- + [{EVENT_WAIT: [run(1, "success")], relay.GATE_CHECK: [run(10, "success")]}],
- (relay.Phase.FINISHED, "success"),
- 16,
- id="a failed read after the grace window keeps waiting",
- ),
- pytest.param(
- [
- {
- OLDER_RACING_WAIT: [run(1, "success", workflow="racing")],
- relay.GATE_CHECK: [run(10, "failure", workflow="racing")],
- }
- ],
- (relay.Phase.FINISHED, "failure"),
- 15,
- id="an older racing event's run stands in for an absent run",
- ),
- pytest.param(
- [
- {
- EVENT_WAIT: [run(1, "cancelled")],
- NEWER_RACING_WAIT: [run(2, "success", workflow="racing")],
- relay.GATE_CHECK: [run(10, "success", workflow="racing")],
- }
- ],
- (relay.Phase.FINISHED, "success"),
- 15,
- id="a newer racing event's run stands in for a cancelled run",
- ),
- pytest.param(
- [
- {
- NEWER_RACING_WAIT: [run(1, "success", workflow="newer")],
- OLDER_RACING_WAIT: [run(2, "success", workflow="older")],
- relay.GATE_CHECK: [run(10, "cancelled", workflow="newer"), run(11, "success", workflow="older")],
- }
- ],
- (relay.Phase.FINISHED, "success"),
- 15,
- id="the next racing event's run stands in when the followed one is cancelled",
- ),
- pytest.param(
- [
- {
- NEWER_RACING_WAIT: [run(1, "success", workflow="newer")],
- OLDER_RACING_WAIT: [run(2, "success", workflow="older")],
- relay.GATE_CHECK: [run(10, "success", workflow="newer"), run(11, "failure", workflow="older")],
- }
- ],
- (relay.Phase.FINISHED, "success"),
- 15,
- id="the newer of two racing runs stands in",
- ),
- pytest.param(
- [
- {
- NEWER_RACING_WAIT: [run(1, "skipped", workflow="declined")],
- OLDER_RACING_WAIT: [run(2, "success", workflow="older")],
- relay.GATE_CHECK: [run(10, "success", workflow="older")],
- }
- ],
- (relay.Phase.FINISHED, "success"),
- 15,
- id="a racing run that declined the hand-off does not stand in",
- ),
- pytest.param(
- [
- {
- relay.wait_check_name(PR, "2026-09-24T09:54:17Z"): [run(1, "success", workflow="earlier")],
- relay.GATE_CHECK: [run(10, "success", workflow="earlier")],
- }
- ],
- (relay.Phase.ABSENT, ""),
- 15,
- id="an event outside the race window does not stand in",
- ),
- pytest.param(
- [{EVENT_WAIT: [run(1, "success")]}],
- (relay.Phase.RUNNING, ""),
- 90,
- id="a running gate hits the overall deadline",
- ),
- ],
-)
-def test_poll_waits_out_a_replacement_before_failing(
- polls: list[dict[str, list[Any]]], expected: tuple[Any, str], min_minutes: int
-) -> None:
- clock = FakeClock()
- result = relay.poll(
- FakeReader(polls),
- EVENT,
- relay.GATE_CHECK,
- deadline_minutes=90,
- absent_minutes=15,
- clock=clock,
- sleep=clock.sleep,
- )
- assert (result.phase, result.state) == expected
- assert clock.now >= min_minutes * 60
-
-
-def test_poll_does_not_use_an_ambiguous_plain_named_wait() -> None:
- clock = FakeClock()
- reader = FakeReader([{PLAIN_WAIT: [run(1, "success")], relay.GATE_CHECK: [run(2, "success")]}])
- result = relay.poll(
- reader, EVENT, relay.GATE_CHECK, deadline_minutes=90, absent_minutes=15, clock=clock, sleep=clock.sleep
- )
- assert result.phase == relay.Phase.ABSENT
- assert PLAIN_WAIT not in reader.reads
-
-
- "result,exit_code,first_line",
- [
- (relay.Progress(relay.Phase.FINISHED, "success"), 0, None),
- (
- relay.Progress(relay.Phase.FINISHED, "failure", "https://depot.dev/orgs/o/workflows/w1?job=j"),
- 1,
- "concluded failure",
- ),
- (relay.Progress(relay.Phase.CANCELLED, "cancelled"), 1, "cancelled its run"),
- (relay.Progress(relay.Phase.DECLINED, "failure"), 1, "declined the hand-off"),
- (relay.Progress(relay.Phase.ABSENT), 1, "started no run"),
- (relay.Progress(relay.Phase.RUNNING), 1, "No Depot verdict"),
- ],
-)
-def test_relay_gate_fails_closed(result: Any, exit_code: int, first_line: str | None) -> None:
- code, lines = relay.relay_gate(result, EVENT, "123")
- assert code == exit_code
- if first_line is None:
- assert lines == []
- else:
- assert first_line in lines[0]
-
-
- "merge_queue,present,absent",
- [
- (False, "--add-label ci-backend-github", "CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT"),
- (True, "CI_BACKEND_DEPOT_MERGE_QUEUE_PERCENT", "--add-label ci-backend-github"),
- ],
-)
-def test_retry_instructions_route_back_to_github_the_way_that_works(
- merge_queue: bool, present: str, absent: str
-) -> None:
- event = dataclasses.replace(EVENT, merge_queue=merge_queue)
- text = "\n".join(relay.retry_instructions(event, "https://depot.dev/orgs/o/workflows/w1", "123"))
- assert present in text
- assert absent not in text
-
-
-def shown(workflow: str, gate: str, executions: int) -> dict[str, Any]:
- return {
- "workflow": {"status": workflow},
- "run": {"run_id": "r1"},
- "executions": [{}] * executions,
- "jobs": [
- {"job_key": "ci-backend.yml:changes", "status": "finished"},
- {"job_key": relay.GATE_JOB_KEY, "status": gate},
- ],
- }
-
-
-class FakeDepot:
- def __init__(self, shows: Sequence[Any]) -> None:
- self._shows = list(shows)
- self._last: dict[str, Any] = {}
- self.sent: list[tuple[str, ...]] = []
-
- def __call__(self, *args: str) -> Any:
- if args[0] != "workflow":
- # Depot refuses a retry of a workflow that still runs.
- assert self._last["workflow"]["status"] not in relay.DEPOT_LIVE_STATES
- self.sent.append(args)
- if self._last.get("refuses_retry"):
- raise subprocess.CalledProcessError(1, "depot")
- return {}
- answer = self._shows.pop(0) if len(self._shows) > 1 else self._shows[0]
- if isinstance(answer, Exception):
- raise answer
- self._last = answer
- return answer
-
-
-RETRY_FAILED = ("retry", "r1", "--workflow", "w1", "--org", "o1", "--failed")
-
-
- "shows,expected,sent",
- [
- pytest.param(
- [
- shown("running", "failed", 1),
- shown("failed", "failed", 1),
- shown("failed", "failed", 1),
- shown("running", "queued", 2),
- shown("failed", "failed", 2),
- ],
- (relay.Phase.FINISHED, "failure"),
- [RETRY_FAILED],
- id="one retry, sent after the workflow finished, and the earlier verdict is not relayed",
- ),
- pytest.param(
- [shown("running", "queued", 2), shown("running", "finished", 2)],
- (relay.Phase.FINISHED, "success"),
- [],
- id="a gate that another retry turned green needs no retry",
- ),
- pytest.param(
- [shown("failed", "failed", 1), shown("running", "queued", 2)],
- (relay.Phase.RUNNING, ""),
- [RETRY_FAILED],
- id="a retry that never finishes hits the deadline",
- ),
- pytest.param([OSError("no depot CLI")], None, [], id="an unreachable Depot gives no verdict"),
- pytest.param(
- [{**shown("failed", "failed", 1), "refuses_retry": True}],
- None,
- [RETRY_FAILED],
- id="a refused retry is not sent again",
- ),
- ],
-)
-def test_retry_relays_only_the_verdict_of_the_new_execution(
- shows: list[Any], expected: tuple[Any, str] | None, sent: list[tuple[str, ...]]
-) -> None:
- depot = FakeDepot(shows)
- clock = FakeClock()
- result = relay.retry_failed_jobs("o1", "w1", depot=depot, clock=clock, sleep=clock.sleep)
- assert (result and (result.phase, result.state)) == expected
- assert depot.sent == sent
-
-
- "rerun,polls,prerequisite,retry_answer,expected,retries",
- [
- pytest.param(False, [run(10, "failure")], [], "success", "failure", 0, id="a first attempt retries nothing"),
- pytest.param(True, [run(10, "failure")], [], "success", "success", 1, id="a failed gate is retried"),
- pytest.param(True, [run(10, "success")], [], "failure", "success", 0, id="a passed gate is relayed"),
- pytest.param(True, [run(10, "failure")], [], None, "failure", 1, id="Depot unreachable"),
- pytest.param(True, [run(10, "cancelled")], [], "success", "success", 1, id="a cancelled gate is retried"),
- pytest.param(True, [None, run(10, "success")], [], "failure", "success", 0, id="a running gate is awaited"),
- pytest.param(
- True,
- [run(10, "cancelled")],
- [run(20, "failure")],
- "success",
- "failure",
- 0,
- id="a failed prerequisite is not retried",
- ),
- ],
-)
-def test_only_a_rerun_with_a_failed_gate_retries_on_depot(
- rerun: bool, polls: list[Any], prerequisite: list[Any], retry_answer: str | None, expected: str, retries: int
-) -> None:
- calls: list[tuple[str, str]] = []
-
- def retry(org: str, workflow: str) -> Any:
- calls.append((org, workflow))
- return retry_answer and relay.Progress(relay.Phase.FINISHED, retry_answer)
-
- reader = FakeReader(
- [
- {
- EVENT_WAIT: [run(1, "success")],
- relay.GATE_CHECK: [gate] if gate else [],
- f"{relay.DEPOT_WORKFLOW} / {relay.PREREQUISITES[0]}": prerequisite,
- }
- for gate in polls
- ]
- )
- result = relay.gate_verdict(reader, EVENT, rerun=rerun, retry=retry)
- assert result.state == expected
- assert calls == [(relay.DEPOT_ORG, "live")] * retries
-
-
-def api_check() -> dict[str, Any]:
- return {
- "id": 1,
- "status": "completed",
- "conclusion": "success",
- "app": {"id": relay.DEPOT_APP_ID},
- "name": relay.GATE_CHECK,
- "head_sha": "abc",
- "pull_requests": [{"number": PR}],
- "details_url": "https://depot.dev/orgs/ntsdt08fpt/workflows/live?job=j1&repo=PostHog%2Fposthog",
- }
-
-
-class FakeResponse:
- def __init__(self, body: bytes, etag: str) -> None:
- self.status = 200
- self.headers = {"ETag": etag}
- self._body = body
-
- def read(self, limit: int = -1) -> bytes:
- return self._body
-
- def __enter__(self) -> "FakeResponse":
- return self
-
- def __exit__(self, *_: object) -> None:
- return None
-
-
-def checks_opener(
- checks: Mapping[int, Mapping[str, Sequence[dict[str, Any]] | None]],
-) -> Callable[[urllib.request.Request, int], FakeResponse]:
- def opener(request: urllib.request.Request, timeout: int) -> FakeResponse:
- query = urllib.parse.parse_qs(urllib.parse.urlparse(request.full_url).query)
- name = query["check_name"][0]
- runs = checks[int(query["app_id"][0])][name]
- if runs is None:
- raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) # type: ignore[arg-type]
- body = [{**check, "name": name, "head_sha": EVENT.sha} for check in runs]
- return FakeResponse(json.dumps({"check_runs": body}).encode(), "")
-
- return opener
-
-
-def test_reader_reuses_its_answer_on_304_and_stops_on_repeated_refusals() -> None:
- sent: list[dict[str, str]] = []
- answers: list[Any] = [
- FakeResponse(json.dumps({"check_runs": [api_check()]}).encode(), '"e1"'),
- urllib.error.HTTPError("url", 304, "Not Modified", {}, None), # type: ignore[arg-type]
- *[urllib.error.HTTPError("url", 403, "Forbidden", {}, None) for _ in range(relay.MAX_REFUSALS)], # type: ignore[arg-type]
- ]
-
- def opener(request: Any, timeout: int) -> Any:
- sent.append(dict(request.header_items()))
- answer = answers.pop(0)
- if isinstance(answer, Exception):
- raise answer
- return answer
-
- reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,))
- assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"]
- assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"]
- assert sent[1]["If-none-match"] == '"e1"'
- for _ in range(relay.MAX_REFUSALS - 1):
- with pytest.raises(relay.ReadFailedError):
- reader.read(relay.GATE_CHECK)
- with pytest.raises(relay.ReadRefusedError):
- reader.read(relay.GATE_CHECK)
-
-
[email protected]("page", [1, 2])
- "error",
- [ConnectionResetError("reset"), http.client.IncompleteRead(b""), ValueError("invalid JSON")],
-)
-def test_reader_retries_interrupted_pages_without_reusing_a_stale_verdict(page: int, error: Exception) -> None:
- payload = api_check()
- answers: list[Any] = [FakeResponse(json.dumps({"check_runs": [payload]}).encode(), '"e1"')]
- if page == 2:
- answers.append(FakeResponse(json.dumps({"check_runs": [payload] * relay.PAGE_SIZE}).encode(), '"e2"'))
- answers += [error, FakeResponse(b'{"check_runs": []}', '"e3"')]
-
- def opener(request: Any, timeout: int) -> FakeResponse:
- answer = answers.pop(0)
- if isinstance(answer, Exception):
- raise answer
- return answer
-
- reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,))
- assert reader.read(relay.GATE_CHECK)[0].state == "success"
- with pytest.raises(relay.ReadFailedError):
- reader.read(relay.GATE_CHECK)
- assert reader.read(relay.GATE_CHECK) == []
- assert not answers
-
-
[email protected]("name", relay.PREREQUISITES)
- "state,workflow,newer",
- [
- ("failure", "live", None),
- ("failure", "other", None),
- ("success", "live", None),
- ("failure", "live", "pending"),
- ("failure", "live", "success"),
- ],
-)
-def test_cancelled_gate_reports_only_current_selected_prerequisite(
- name: str, state: str, workflow: str, newer: str | None
-) -> None:
- roots = [run(2, state, workflow)]
- if newer:
- roots.append(run(3, newer, workflow))
- clock = FakeClock()
- result = relay.poll(
- FakeReader(
- [
- {
- EVENT_WAIT: [run(1, "success")],
- relay.GATE_CHECK: [run(5, "cancelled")],
- f"{relay.DEPOT_WORKFLOW} / {name}": roots,
- }
- ]
- ),
- EVENT,
- relay.GATE_CHECK,
- deadline_minutes=90,
- absent_minutes=15,
- clock=clock,
- sleep=clock.sleep,
- )
- code, lines = relay.relay_gate(result, EVENT, "123")
- assert code == 1
- if state == "failure" and workflow == "live" and newer is None:
- assert clock.now == 0
- assert lines == [f"::error::{name} failed on Depot (check 2). Push a fix; a retry will not help."]
- assert result.root_check_id == 2
- else:
- assert result.phase == relay.Phase.CANCELLED
- assert clock.now == 900
-
-
[email protected]("name", relay.PREREQUISITES)
-def test_failed_gate_keeps_retry_options_after_a_prerequisite_failure(name: str) -> None:
- # Without Depot's self-cancel, the prerequisite may have failed on a retryable setup step.
- clock = FakeClock()
- result = relay.poll(
- FakeReader(
- [
- {
- EVENT_WAIT: [run(1, "success")],
- relay.GATE_CHECK: [run(5, "failure")],
- f"{relay.DEPOT_WORKFLOW} / {name}": [run(2, "failure")],
- }
- ]
- ),
- EVENT,
- relay.GATE_CHECK,
- deadline_minutes=90,
- absent_minutes=15,
- clock=clock,
- sleep=clock.sleep,
- )
- code, lines = relay.relay_gate(result, EVENT, "123")
- assert code == 1
- assert not result.root_failure
- assert not any("a retry will not help" in line for line in lines)
-
-
- "field,value",
- [
- ("app", {"id": 99}),
- ("app", None),
- ("name", "other check"),
- ("head_sha", "other"),
- ("pull_requests", [{"number": PR + 1}]),
- ("details_url", "https://depot.dev/orgs/other/workflows/live"),
- ("details_url", "https://depot.dev.example.com/orgs/ntsdt08fpt/workflows/live"),
- ("details_url", "https://[email protected]/orgs/ntsdt08fpt/workflows/live"),
- ("details_url", "http://depot.dev/orgs/ntsdt08fpt/workflows/live"),
- ],
-)
-def test_reader_rejects_wrong_identity(field: str, value: Any) -> None:
- payload = {**api_check(), field: value}
- reader = relay.CheckRunReader(
- "PostHog/posthog",
- "abc",
- "token",
- pr_number=PR,
- opener=lambda *a, **kw: FakeResponse(json.dumps({"check_runs": [payload]}).encode(), ""),
- )
- assert reader.read(relay.GATE_CHECK) == []
-
-
-def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None:
- answer = json.dumps({"check_runs": [{**api_check(), "app": None}, api_check()]}).encode()
- reader = relay.CheckRunReader(
- "PostHog/posthog", "abc", "token", pr_number=PR, opener=lambda *a, **kw: FakeResponse(answer, "")
- )
- assert len(reader.read(relay.GATE_CHECK)) == 1
-
-
- "mirror_gate,depot_gate,expected",
- [
- pytest.param(
- [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")],
- [],
- (relay.Phase.FINISHED, "success"),
- id="Depot's copies lag",
- ),
- pytest.param(
- [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")],
- [(9, "failure", "w1")],
- (relay.Phase.FINISHED, "success"),
- id="a late copy of an older attempt",
- ),
- pytest.param(
- [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")],
- [(8, "success", "w1"), (9, "failure", "w1")],
- (relay.Phase.FINISHED, "success"),
- id="late copies of both attempts",
- ),
- pytest.param(
- [(2, "failure", "w1", "a1"), (5, "failure", "w1", "a2")],
- [(4, "failure", "w1"), (9, "failure", "w1")],
- (relay.Phase.FINISHED, "failure"),
- id="a retry that failed again",
- ),
- pytest.param(
- [(2, "success", "w1", "a1")],
- [(4, "success", "w1"), (9, "failure", "w1")],
- (relay.Phase.FINISHED, "failure"),
- id="a newer attempt the mirror missed",
- ),
- pytest.param(
- [(2, "failure", "w1", "a1")],
- [(4, "failure", "w1"), (9, "in_progress", "w1")],
- (relay.Phase.RUNNING, "in_progress"),
- id="a running retry the mirror has not posted",
- ),
- pytest.param(
- [(5, "success", "w1", "a2")],
- [(4, "cancelled", "w1")],
- (relay.Phase.FINISHED, "success"),
- id="a replacement for a job cancelled before it started",
- ),
- pytest.param(
- [(5, "success", "w1", "a2")],
- [(4, "cancelled", "w1"), (9, "in_progress", "w1")],
- (relay.Phase.RUNNING, "in_progress"),
- id="a replacement Depot has not finished after a cancelled job",
- ),
- pytest.param(
- [(5, "success", "other", "a1")],
- [(9, "failure", "w1")],
- (relay.Phase.FINISHED, "failure"),
- id="another workflow's mirrored gate",
- ),
- pytest.param(
- [(5, "success", "other", "a1")],
- [],
- (relay.Phase.RUNNING, ""),
- id="only another workflow's mirrored gate",
- ),
- pytest.param([(5, "success", "w1", "a1")], None, None, id="Depot's app read fails"),
- ],
-)
-def test_relay_reads_the_current_attempt_across_apps(
- mirror_gate: list[tuple[int, str, str, str]],
- depot_gate: list[tuple[int, str, str]] | None,
- expected: tuple[Any, str] | None,
-) -> None:
- mirror = {
- EVENT_WAIT: [api_run(1, "success", "w1", "w")],
- relay.GATE_CHECK: [],
- relay.MIRRORED_GATE_CHECK: [api_run(*c) for c in mirror_gate],
- }
- depot = {
- EVENT_WAIT: [api_run(3, "success", "w1")],
- relay.GATE_CHECK: None if depot_gate is None else [api_run(*c) for c in depot_gate],
- }
-
- reader = relay.CheckRunReader(
- EVENT.repo, EVENT.sha, "token", opener=checks_opener({relay.MIRROR_APP_ID: mirror, relay.DEPOT_APP_ID: depot})
- )
- wait = relay.newest_live(reader.read(EVENT_WAIT))
- if expected is None:
- with pytest.raises(relay.ReadFailedError):
- reader.read(relay.GATE_CHECK)
- return
- current = relay.progress(wait, reader.read(relay.GATE_CHECK))
- assert (current.phase, current.state) == expected
-
-
- "posted,unreadable,expected",
- [
- pytest.param([(relay.GATE_CHECK, "success")], None, "success", id="an older workflow revision"),
- pytest.param(
- [(relay.GATE_CHECK, "success")],
- relay.MIRRORED_GATE_CHECK,
- None,
- id="a failed read of the mirror's new name",
- ),
- pytest.param(
- [(relay.MIRRORED_GATE_CHECK, "success")],
- relay.GATE_CHECK,
- None,
- id="a failed read of the mirror's old name",
- ),
- pytest.param(
- [(relay.GATE_CHECK, "success"), (relay.MIRRORED_GATE_CHECK, "failure")],
- None,
- "failure",
- id="a newer failed attempt under the new name",
- ),
- pytest.param(
- [(relay.MIRRORED_GATE_CHECK, "success"), (relay.GATE_CHECK, "failure")],
- None,
- "failure",
- id="a newer failed attempt under the old name",
- ),
- ],
-)
-def test_reader_reads_the_mirrored_gate_under_both_names(
- posted: Sequence[tuple[str, str]], unreadable: str | None, expected: str | None
-) -> None:
- mirror: dict[str, list[dict[str, Any]] | None] = {relay.GATE_CHECK: [], relay.MIRRORED_GATE_CHECK: []}
- for index, (name, state) in enumerate(posted, start=1):
- mirror[name] = [api_run(index, state, "w1", f"a{index}")]
- if unreadable:
- mirror[unreadable] = None
- reader = relay.CheckRunReader(
- EVENT.repo,
- EVENT.sha,
- "token",
- opener=checks_opener({relay.MIRROR_APP_ID: mirror}),
- app_ids=(relay.MIRROR_APP_ID,),
- )
- if expected is None:
- with pytest.raises(relay.ReadFailedError):
- reader.read(relay.GATE_CHECK)
- return
- current = relay.current_check(reader.read(relay.GATE_CHECK), "w1")
- assert current is not None
- assert current.state == expected
diff --git a/.github/scripts/test_ci_backend_route.py b/.github/scripts/test_ci_backend_route.py
deleted file mode 100644
index 24c2c8a28f46..000000000000
--- a/.github/scripts/test_ci_backend_route.py
+++ /dev/null
@@ -1,291 +0,0 @@
-import json
-import urllib.error
-import importlib.util
-from collections.abc import Sequence
-from datetime import UTC, datetime
-from pathlib import Path
-from typing import Any
-
-import pytest
-
-SCRIPT_PATH = Path(__file__).with_name("ci_backend_route.py")
-SPEC = importlib.util.spec_from_file_location("ci_backend_route", SCRIPT_PATH)
-assert SPEC is not None
-assert SPEC.loader is not None
-route = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(route)
-
-EVENT_AT = "2026-09-30T05:44:10Z"
-EVENT_EPOCH = datetime.strptime(EVENT_AT, route.EVENT_TIME).replace(tzinfo=UTC).timestamp()
-
-
-def pr(
- percent: int = 25, number: int | None = 125, labels: Sequence[str] = (), fork: bool = False, draft: bool = False
-) -> Any:
- return route.decide("pull_request", percent, number, list(labels), fork, draft)
-
-
[email protected]("number,bucket", [(0, 40), (7, 30), (124, 94), (125, 19), (103100, 33)])
-def test_bucket_is_a_fixed_hash_of_the_pr_number(number: int, bucket: int) -> None:
- assert route.bucket_of(number) == bucket
- assert pr(number=number, percent=bucket).engine == "github"
- assert pr(number=number, percent=bucket + 1).engine == "depot"
-
-
-def test_consecutive_prs_spread_across_buckets() -> None:
- routed = [pr(number=number, percent=5).engine == "depot" for number in range(100000, 101000)]
- assert 30 <= sum(routed) <= 70
- assert not any(all(routed[i : i + 5]) for i in range(len(routed) - 4))
-
-
- "labels,fork,draft,expected",
- [
- (["ci-backend-github"], False, False, "github"),
- (["ci-backend-depot"], False, False, "depot"),
- (["ci-backend-depot", "ci-backend-github"], False, False, "github"),
- (["ci-backend-depot"], True, False, "github"),
- (["ci-backend-depot", "no-ci"], False, True, "github"),
- (["no-ci"], False, False, "depot"),
- ],
-)
-def test_overrides(labels: list[str], fork: bool, draft: bool, expected: str) -> None:
- assert pr(labels=labels, fork=fork, draft=draft).engine == expected
-
-
[email protected]("event", ["workflow_dispatch", "push", "schedule", "merge_group"])
-def test_non_pull_request_events_stay_on_github(event: str) -> None:
- assert route.decide(event, 100, None, ["ci-backend-depot"], False, False, "success").engine == "github"
-
-
- "head_ref,labels,percent,queue_percent,prior,expected",
- [
- ("trunk-merge/pr-1/x", ["ci-backend-depot"], 100, 0, None, "github"),
- ("trunk-merge/pr-1/x", ["ci-backend-github"], 0, 100, None, "depot"),
- ("trunk-merge/pr-1/x", [], 0, 0, "success", "depot"),
- ("feature/trunk-merge", [], 100, 0, None, "depot"),
- ],
-)
-def test_merge_queue_batches_route_by_their_own_percent(
- head_ref: str, labels: list[str], percent: int, queue_percent: int, prior: str | None, expected: str
-) -> None:
- decision = route.decide("pull_request", percent, 124, labels, False, True, prior, head_ref, queue_percent)
- assert decision.engine == expected
-
-
-def test_missing_pr_number_stays_on_github() -> None:
- assert pr(number=None, percent=100).engine == "github"
-
-
- "prior,labels,percent,expected",
- [
- ("success", ["ci-backend-github"], 0, "depot"),
- ("skipped", ["ci-backend-depot"], 100, "github"),
- ("cancelled", ["ci-backend-depot"], 0, "depot"),
- (None, [], 100, "depot"),
- ],
-)
-def test_earlier_run_of_the_commit_wins(prior: str | None, labels: list[str], percent: int, expected: str) -> None:
- assert route.decide("pull_request", percent, 124, labels, False, False, prior).engine == expected
-
-
-def check(run_id: int, status: str, conclusion: str | None, pr_number: int = 124) -> dict[str, Any]:
- return {"id": run_id, "status": status, "conclusion": conclusion, "pull_requests": [{"number": pr_number}]}
-
-
- "check_runs,expected",
- [
- ([check(1, "completed", "success"), check(2, "queued", None)], "success"),
- ([check(1, "completed", "success"), check(2, "completed", "cancelled")], "success"),
- ([check(1, "completed", "success"), check(2, "completed", "skipped")], "success"),
- ([check(1, "completed", "skipped"), check(2, "completed", "cancelled")], "skipped"),
- ([check(1, "completed", "cancelled"), check(2, "completed", "failure")], "failure"),
- ([check(1, "completed", "success", pr_number=125)], None),
- ([{"id": 1, "status": "completed", "conclusion": "success", "pull_requests": []}], None),
- ([], None),
- ],
-)
-def test_handoff_conclusion_keeps_the_engine_that_already_ran_this_pull_request(
- check_runs: list[dict[str, Any]], expected: str | None
-) -> None:
- assert route.handoff_conclusion(check_runs, 124) == expected
-
-
-class FakeResponse:
- def __init__(self, payload: Any) -> None:
- self.payload = payload
-
- def read(self) -> bytes:
- return json.dumps(self.payload).encode()
-
- def __enter__(self) -> "FakeResponse":
- return self
-
- def __exit__(self, *exc: object) -> None:
- return None
-
-
-def http_error(code: int) -> urllib.error.HTTPError:
- return urllib.error.HTTPError("https://api.github.com", code, "boom", {}, None) # type: ignore[arg-type]
-
-
-def opener_from(answers: list[Any]) -> Any:
- def opener(request: Any, timeout: int) -> Any:
- answer = answers.pop(0)
- if isinstance(answer, Exception):
- raise answer
- return FakeResponse(answer)
-
- return opener
-
-
-def test_fetch_retries_a_server_error_then_returns_the_checks(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(route.time, "sleep", lambda seconds: None)
- checks = [check(1, "completed", "success")]
- opener = opener_from([http_error(502), {"check_runs": checks}])
- assert route.fetch_handoff_checks("PostHog/posthog", "abc", "t", opener=opener) == checks
-
-
- "answers,attempts_used",
- [([http_error(403)], 1), ([http_error(503)] * 3, 3), ([urllib.error.URLError("down")] * 3, 3)],
-)
-def test_fetch_fails_closed_without_retrying_client_errors(
- answers: list[Exception], attempts_used: int, monkeypatch: pytest.MonkeyPatch
-) -> None:
- monkeypatch.setattr(route.time, "sleep", lambda seconds: None)
- remaining = list(answers)
- with pytest.raises(route.HandoffReadError):
- route.fetch_handoff_checks("PostHog/posthog", "abc", "t", opener=opener_from(remaining))
- assert len(answers) - len(remaining) == attempts_used
-
-
- "raw,expected",
- [(None, 0), ("", 0), ("abc", 0), ("-5", 0), ("42", 42), ("250", 100), (" 7 ", 7), ("²", 0), ("9" * 5000, 0)],
-)
-def test_parse_percent_fails_closed(raw: str | None, expected: int) -> None:
- assert route.parse_percent(raw) == expected
-
-
- "labels,started,head_ref,expected",
- [
- (json.dumps(["other"]), True, "", "engine=depot\nreason=bucket 30 < 50%\n"),
- ("null", True, "", "engine=depot\nreason=bucket 30 < 50%\n"),
- ("", True, "", "engine=depot\nreason=bucket 30 < 50%\n"),
- ("[]", False, "", "engine=github\nreason=Depot CI started no run for this event\n"),
- (json.dumps(["ci-backend-depot"]), False, "", "engine=github\nreason=Depot CI started no run for this event\n"),
- ("[]", True, "trunk-merge/pr-1/x", "engine=github\nreason=merge queue bucket 30 >= 20%\n"),
- ],
-)
-def test_main_writes_outputs(
- labels: str, started: bool, head_ref: str, expected: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- output = tmp_path / "out"
- monkeypatch.setenv("GITHUB_OUTPUT", str(output))
- monkeypatch.setenv("EVENT", "pull_request")
- monkeypatch.setenv("PERCENT", "50")
- monkeypatch.setenv("MERGE_QUEUE_PERCENT", "20")
- monkeypatch.setenv("HEAD_REF", head_ref)
- monkeypatch.setenv("PR_NUMBER", "7")
- monkeypatch.setenv("LABELS", labels)
- monkeypatch.setattr(route, "fetch_handoff_checks", lambda repo, sha, token: [])
- monkeypatch.setattr(route, "depot_started", lambda *_: started)
- monkeypatch.setenv("REPO", "PostHog/posthog")
- monkeypatch.setenv("SHA", "abc")
- monkeypatch.setenv("GH_TOKEN", "t")
- assert route.main() == 0
- assert output.read_text() == expected
-
-
[email protected]("percent", ["", "0", "5"])
-def test_main_keeps_a_handed_off_commit_on_depot_after_rollback(
- percent: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- def fetch(repo: str, sha: str, token: str) -> list[dict[str, Any]]:
- return [{"id": 1, "status": "completed", "conclusion": "success", "pull_requests": [{"number": 7}]}]
-
- output = tmp_path / "out"
- monkeypatch.setenv("GITHUB_OUTPUT", str(output))
- monkeypatch.setenv("EVENT", "pull_request")
- monkeypatch.setenv("PERCENT", percent)
- monkeypatch.setenv("PR_NUMBER", "7")
- monkeypatch.setenv("LABELS", "[]")
- monkeypatch.setenv("REPO", "PostHog/posthog")
- monkeypatch.setenv("SHA", "abc")
- monkeypatch.setenv("GH_TOKEN", "t")
- monkeypatch.setattr(route, "fetch_handoff_checks", fetch)
- monkeypatch.setattr(route, "depot_started", lambda *_: False)
- assert route.main() == 0
- assert output.read_text().startswith("engine=depot\n")
-
-
- "percent,labels",
- [
- ("50", "[]"),
- ("", json.dumps(["ci-backend-depot"])),
- ("5", "[]"),
- ("0", "[]"),
- ("", "[]"),
- ("50", json.dumps(["ci-backend-github"])),
- ],
-)
-def test_main_never_switches_engines_on_an_unreadable_handoff(
- percent: str, labels: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- def failing(repo: str, sha: str, token: str) -> list[dict[str, Any]]:
- raise route.HandoffReadError("boom")
-
- output = tmp_path / "out"
- monkeypatch.setenv("GITHUB_OUTPUT", str(output))
- monkeypatch.setenv("EVENT", "pull_request")
- monkeypatch.setenv("PERCENT", percent)
- monkeypatch.setenv("PR_NUMBER", "7")
- monkeypatch.setenv("LABELS", labels)
- monkeypatch.setenv("REPO", "PostHog/posthog")
- monkeypatch.setenv("SHA", "abc")
- monkeypatch.setenv("GH_TOKEN", "t")
- monkeypatch.setattr(route, "fetch_handoff_checks", failing)
- monkeypatch.setattr(route.time, "sleep", lambda seconds: None)
- assert route.main() == 1
- assert not output.exists()
-
-
-class FakeReader:
- def __init__(self, polls: list[bool | Exception]) -> None:
- self.polls = polls
-
- def read(self, name: str) -> list[str]:
- assert name == f"Backend CI on Depot / Depot run started (PR 7, event {EVENT_AT})"
- answer = self.polls.pop(0) if len(self.polls) > 1 else self.polls[0]
- if isinstance(answer, Exception):
- raise answer
- return ["check"] if answer else []
-
-
- "polls,started,routed_after",
- [
- ([True], True, 15),
- ([False, False, True], True, 15),
- ([route.ReadFailedError("boom"), True], True, 15),
- ([False], False, 15),
- ([route.ReadRefusedError("refused")], False, 15),
- ([False], False, 400),
- ],
-)
-def test_depot_started_waits_for_the_started_check_of_the_event(
- polls: list[bool | Exception], started: bool, routed_after: int
-) -> None:
- now = [EVENT_EPOCH + routed_after]
-
- def sleep(seconds: float) -> None:
- now[0] += seconds
-
- assert route.depot_started(FakeReader(polls), 7, EVENT_AT, clock=lambda: now[0], sleep=sleep) is started
- assert now[0] <= max(EVENT_EPOCH + route.DEPOT_START_SECONDS + route.DEPOT_POLL_SECONDS, EVENT_EPOCH + routed_after)
diff --git a/.github/scripts/test_ci_flake_overseer.py b/.github/scripts/test_ci_flake_overseer.py
deleted file mode 100644
index fc3ad7e0de53..000000000000
--- a/.github/scripts/test_ci_flake_overseer.py
+++ /dev/null
@@ -1,359 +0,0 @@
-from __future__ import annotations
-
-import json
-
-import pytest
-
-import ci_flake_overseer
-from ci_flake_overseer import (
- DECISION_EVENT,
- OUTCOME_EVENT,
- Decision,
- Job,
- Step,
- WorkflowRun,
- build_decision_events,
- capture_events,
- classify_job,
- is_test_job_failure,
- report_rerun_outcomes,
- rerun_outcome_label,
- workflow_run_from_object,
-)
-
-
-def make_job(
- name: str,
- *,
- run_attempt: int = 1,
- conclusion: str = "failure",
- failed_step: str = "Run Core tests",
- started_at: str = "2026-06-05T21:00:00Z",
-) -> Job:
- return Job(
- id=123,
- name=name,
- conclusion=conclusion,
- run_attempt=run_attempt,
- html_url="https://github.com/PostHog/posthog/actions/runs/1/job/123",
- started_at=started_at,
- steps=(Step(name=failed_step, conclusion="failure"),),
- )
-
-
-def make_workflow_run(
- run_attempt: int = 1, *, head_branch: str = "", event: str = "pull_request", pr_number: int | None = 12345
-) -> WorkflowRun:
- return WorkflowRun(
- id=999,
- workflow_id=42,
- name="Backend CI",
- conclusion="failure",
- head_sha="abc123",
- run_attempt=run_attempt,
- html_url="https://github.com/PostHog/posthog/actions/runs/999",
- head_branch=head_branch,
- event=event,
- pr_number=pr_number,
- )
-
-
- ("job", "expected_action", "expected_reason_fragment", "expected_failure_class"),
- [
- pytest.param(
- make_job("Repo checks (depot-ubuntu-latest)", failed_step="Check module boundaries (tach)"),
- "skip deterministic",
- "deterministic",
- "repo-checks",
- id="deterministic-repo-checks",
- ),
- pytest.param(
- make_job("Validate OpenAPI types", failed_step="Check and update OpenAPI types"),
- "skip deterministic",
- "deterministic",
- "openapi",
- id="deterministic-openapi",
- ),
- pytest.param(
- make_job("Frontend lint", failed_step="Lint with Oxlint"),
- "skip deterministic",
- "deterministic",
- "lint",
- id="deterministic-lint",
- ),
- pytest.param(
- make_job("Validate migrations", failed_step="Check migrations"),
- "skip deterministic",
- "deterministic",
- "migrations",
- id="deterministic-migrations",
- ),
- pytest.param(
- make_job("Django tests", failed_step="Product facade enforcement"),
- "skip deterministic",
- "deterministic",
- "module-boundaries",
- id="deterministic-via-step-rule",
- ),
- pytest.param(
- make_job("Build and deploy", failed_step="Compile assets"),
- "skip non-test",
- "not an allowlisted test runner",
- None,
- id="non-test-job",
- ),
- pytest.param(
- make_job("Django tests - Temporal (1/1)", failed_step="Run Temporal tests"),
- "observe",
- "test job failure",
- None,
- id="observed-test-job",
- ),
- ],
-)
-def test_classify_job(
- job: Job, expected_action: str, expected_reason_fragment: str, expected_failure_class: str | None
-) -> None:
- decision = classify_job(job)
-
- assert decision.action == expected_action
- assert expected_reason_fragment in decision.reason
- assert decision.failure_class == expected_failure_class
-
-
- ("job_name", "failed_step", "expected"),
- [
- pytest.param("Django tests - Temporal (1/1)", "Run Core tests", True, id="test-job-and-test-step"),
- pytest.param(
- "Django tests - Temporal (1/1)", "Stop containers", True, id="test-job-nontest-step-falls-back-to-name"
- ),
- pytest.param("Build and deploy", "Run Playwright tests", True, id="nontest-job-test-step"),
- pytest.param("Build and deploy", "Compile assets", False, id="nontest-job-and-nontest-step"),
- ],
-)
-def test_is_test_job_failure(job_name: str, failed_step: str, expected: bool) -> None:
- assert is_test_job_failure(make_job(job_name, failed_step=failed_step)) is expected
-
-
-def test_build_decision_events_one_event_per_decision() -> None:
- decisions = (
- classify_job(make_job("Django tests - Temporal (1/1)")),
- classify_job(make_job("Build and deploy", failed_step="Compile assets")),
- classify_job(make_job("Validate OpenAPI types", failed_step="Check and update OpenAPI types")),
- )
-
- events = build_decision_events("PostHog/posthog", make_workflow_run(), decisions)
-
- assert [event["event"] for event in events] == [DECISION_EVENT, DECISION_EVENT, DECISION_EVENT]
- assert all(event["distinct_id"] == "PostHog/posthog" for event in events)
- first = events[0]["properties"]
- assert first["action"] == "observe"
- assert first["workflow_name"] == "Backend CI"
- assert first["job_name"] == "Django tests - Temporal (1/1)"
- assert first["$groups"] == {"workflow_run": "999"}
- assert first["classified_via"] == "job_name"
- assert first["failure_class"] is None
- assert first["pr_number"] == 12345
- assert first["job_conclusion"] == "failure"
- assert first["failed_steps"] == ["Run Core tests"]
- assert first["$insert_id"] == "decision:999:1:123"
- assert events[1]["properties"]["action"] == "skip non-test"
- assert events[1]["properties"]["classified_via"] is None
- assert events[2]["properties"]["action"] == "skip deterministic"
- assert events[2]["properties"]["failure_class"] == "openapi"
-
-
- ("pull_requests", "expected"),
- [
- pytest.param([{"number": 65581}], 65581, id="same-repo-pr"),
- pytest.param([], None, id="fork-pr-empty-list"),
- pytest.param(None, None, id="field-absent"),
- ],
-)
-def test_workflow_run_pr_number(pull_requests: list[dict[str, object]] | None, expected: int | None) -> None:
- raw: dict[str, object] = {"id": 1, "workflow_id": 2, "name": "Backend CI"}
- if pull_requests is not None:
- raw["pull_requests"] = pull_requests
-
- assert workflow_run_from_object(raw).pr_number == expected
-
-
- ("conclusion", "expected"),
- [
- pytest.param("success", "cleared", id="success-cleared"),
- pytest.param("failure", "still_failing", id="failure-still-failing"),
- pytest.param("timed_out", "still_failing", id="timed-out-still-failing"),
- pytest.param(None, "unknown", id="missing-unknown"),
- pytest.param("cancelled", "unknown", id="cancelled-unknown"),
- ],
-)
-def test_rerun_outcome_label(conclusion: str | None, expected: str) -> None:
- assert rerun_outcome_label(conclusion) == expected
-
-
-def test_report_rerun_outcomes_empty_on_first_attempt() -> None:
- assert report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=1)) == []
-
-
- ("current_conclusion", "expected_outcome"),
- [
- pytest.param("success", "cleared", id="rerun-cleared"),
- pytest.param("failure", "still_failing", id="rerun-still-failing"),
- pytest.param("timed_out", "still_failing", id="rerun-timed-out"),
- ],
-)
-def test_report_rerun_outcomes_attributes_prior_test_failure(
- monkeypatch: pytest.MonkeyPatch, current_conclusion: str, expected_outcome: str
-) -> None:
- job_name = "Django tests - Temporal (1/1)"
-
- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:
- if attempt == 1:
- return (make_job(job_name, conclusion="failure", started_at="2026-06-05T21:00:00Z"),)
- # Advanced started_at: this attempt genuinely re-executed the job.
- return (make_job(job_name, conclusion=current_conclusion, started_at="2026-06-05T22:00:00Z"),)
-
- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)
-
- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))
-
- assert len(events) == 1
- assert events[0]["event"] == OUTCOME_EVENT
- props = events[0]["properties"]
- assert props["outcome"] == expected_outcome
- assert props["job_name"] == job_name
- assert props["prior_conclusion"] == "failure"
- assert props["prior_attempt"] == 1
- assert props["attempt"] == 2
- assert props["$insert_id"] == f"outcome:999:2:{job_name}"
-
-
-def test_report_rerun_outcomes_not_rerun_when_started_at_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:
- # The regression this fix targets: a failed job carried over (never re-executed) keeps its prior
- # started_at and conclusion. It must read as `not_rerun`, never `still_failing`.
- job_name = "Django tests - Temporal (1/1)"
-
- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:
- return (make_job(job_name, conclusion="failure", started_at="2026-06-05T21:00:00Z"),)
-
- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)
-
- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))
-
- assert len(events) == 1
- assert events[0]["properties"]["outcome"] == "not_rerun"
-
-
-def test_report_rerun_outcomes_unknown_when_job_absent(monkeypatch: pytest.MonkeyPatch) -> None:
- job_name = "Django tests - Temporal (1/1)"
-
- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:
- if attempt == 1:
- return (make_job(job_name, conclusion="failure"),)
- return () # job missing from the re-run attempt
-
- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)
-
- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))
-
- assert len(events) == 1
- assert events[0]["properties"]["outcome"] == "unknown"
-
-
-def test_report_rerun_outcomes_unknown_when_current_name_ambiguous(monkeypatch: pytest.MonkeyPatch) -> None:
- job_name = "Django tests - Temporal (1/1)"
-
- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:
- if attempt == 1:
- return (make_job(job_name, conclusion="failure"),)
- # Two current jobs share a name: ambiguous, so it can't be paired and must read as unknown.
- return (
- make_job(job_name, conclusion="success", started_at="2026-06-05T22:00:00Z"),
- make_job(job_name, conclusion="failure", started_at="2026-06-05T22:00:00Z"),
- )
-
- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)
-
- events = report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2))
-
- assert len(events) == 1
- assert events[0]["properties"]["outcome"] == "unknown"
-
-
- ("head_branch", "event", "expected_is_master"),
- [
- pytest.param("master", "push", True, id="master-is-master"),
- pytest.param("main", "push", True, id="main-is-master"),
- pytest.param("some-feature", "pull_request", False, id="pr-branch-not-master"),
- ],
-)
-def test_build_decision_events_is_master(head_branch: str, event: str, expected_is_master: bool) -> None:
- decisions = (classify_job(make_job("Django tests - Temporal (1/1)")),)
-
- events = build_decision_events(
- "PostHog/posthog", make_workflow_run(head_branch=head_branch, event=event), decisions
- )
-
- props = events[0]["properties"]
- assert props["is_master"] is expected_is_master
- assert props["trigger_event"] == event
-
-
-def test_report_rerun_outcomes_ignores_non_test_jobs(monkeypatch: pytest.MonkeyPatch) -> None:
- def fake_fetch_jobs(repo: str, run_id: int, attempt: int, *, strict: bool = False) -> tuple[Job, ...]:
- return (make_job("Build and deploy", conclusion="failure", failed_step="Compile assets"),)
-
- monkeypatch.setattr(ci_flake_overseer, "fetch_jobs", fake_fetch_jobs)
-
- assert report_rerun_outcomes("PostHog/posthog", make_workflow_run(run_attempt=2)) == []
-
-
-def _raise_if_called(*args: object, **kwargs: object) -> None:
- raise AssertionError("telemetry should not POST")
-
-
-def test_capture_events_noop_without_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(ci_flake_overseer.urllib.request, "urlopen", _raise_if_called)
-
- capture_events("", "https://us.i.posthog.com", [{"event": DECISION_EVENT}])
-
-
-def test_capture_events_noop_without_events(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(ci_flake_overseer.urllib.request, "urlopen", _raise_if_called)
-
- capture_events("phc_test", "https://us.i.posthog.com", [])
-
-
-def test_capture_events_posts_batch(monkeypatch: pytest.MonkeyPatch) -> None:
- captured: dict[str, object] = {}
-
- class FakeResponse:
- def __enter__(self) -> FakeResponse:
- return self
-
- def __exit__(self, *args: object) -> bool:
- return False
-
- def fake_urlopen(request: object, timeout: object = None) -> FakeResponse:
- captured["url"] = request.full_url # type: ignore[attr-defined]
- captured["body"] = json.loads(request.data.decode()) # type: ignore[attr-defined]
- return FakeResponse()
-
- monkeypatch.setattr(ci_flake_overseer.urllib.request, "urlopen", fake_urlopen)
- events = [{"event": DECISION_EVENT, "distinct_id": "PostHog/posthog", "properties": {}}]
-
- capture_events("phc_test", "https://us.i.posthog.com/", events)
-
- assert captured["url"] == "https://us.i.posthog.com/batch/"
- assert captured["body"] == {"api_key": "phc_test", "batch": events}
-
-
-def test_decision_is_minimal_dataclass() -> None:
- decision = Decision(action="observe", reason="x", job=make_job("Django tests - Temporal (1/1)"))
- assert decision.action == "observe"
diff --git a/.github/scripts/test_ci_migration_selection.py b/.github/scripts/test_ci_migration_selection.py
index 5b29c0116703..3105ef0847cf 100644
--- a/.github/scripts/test_ci_migration_selection.py
+++ b/.github/scripts/test_ci_migration_selection.py
@@ -38,13 +38,16 @@ def commit_file(repo: Path, name: str, content: str) -> str:
def commit_on_master(repo: Path, master: str, name: str, content: str) -> None:
git(repo, "checkout", "-q", "master")
commit_file(repo, name, content)
- git(repo, "update-ref", f"refs/remotes/{master}", "HEAD")
+ git(repo, "update-ref", f"refs/remotes/{MASTER}", "HEAD")
git(repo, "checkout", "-q", "topic")
+MASTER = "upstream/master"
+
+
@functools.cache
-def list_step(engine: str) -> str:
- workflow = yaml.safe_load((ROOT / engine / "workflows/ci-backend.yml").read_text())
+def list_step() -> str:
+ workflow = yaml.safe_load((ROOT / ".depot/workflows/ci-backend.yml").read_text())
return next(
step["run"]
for job in workflow["jobs"].values()
@@ -53,7 +56,6 @@ def list_step(engine: str) -> str:
)
[email protected]("engine,master", [(".github", "origin/master"), (".depot", "upstream/master")])
@pytest.mark.parametrize(
"case,expected",
[
@@ -69,8 +71,6 @@ def list_step(engine: str) -> str:
],
)
def test_lists_only_this_prs_migrations(
- engine: str,
- master: str,
case: str,
expected: tuple[list[str], list[str]] | None,
tmp_path: Path,
@@ -83,7 +83,7 @@ def test_lists_only_this_prs_migrations(
git(repo, "config", "commit.gpgsign", "false")
base = commit_file(repo, "0001_base.py", "base\n")
commit_file(repo, "0002_master.py", "master\n")
- git(repo, "update-ref", f"refs/remotes/{master}", "HEAD")
+ git(repo, "update-ref", f"refs/remotes/{MASTER}", "HEAD")
git(repo, "checkout", "-qb", "topic")
if case == "based on master":
@@ -92,10 +92,10 @@ def test_lists_only_this_prs_migrations(
elif case == "stale base":
commit_file(repo, "0003_own.py", "own\n")
elif case == "master adds a similar migration later":
- commit_on_master(repo, master, "0003_master.py", MIGRATION_TEMPLATE.format(table="master_table"))
+ commit_on_master(repo, MASTER, "0003_master.py", MIGRATION_TEMPLATE.format(table="master_table"))
commit_file(repo, "0003_own.py", MIGRATION_TEMPLATE.format(table="own_table"))
elif case == "master edits a migration the branch carries":
- commit_on_master(repo, master, "0002_master.py", "master fixed\n")
+ commit_on_master(repo, MASTER, "0002_master.py", "master fixed\n")
elif case == "edits an existing migration":
commit_file(repo, "0001_base.py", "modified\n")
elif case == "base missing from the clone":
@@ -107,16 +107,15 @@ def test_lists_only_this_prs_migrations(
["git", "clone", "-q", "--depth=1", "--no-single-branch", "-b", "topic", source.as_uri(), str(repo)],
check=True,
)
- if engine == ".depot":
- git(repo, "update-ref", f"refs/remotes/{master}", "refs/remotes/origin/master")
- git(repo, "remote", "remove", "origin")
+ git(repo, "update-ref", f"refs/remotes/{MASTER}", "refs/remotes/origin/master")
+ git(repo, "remote", "remove", "origin")
elif case == "unknown base":
base = "0" * 40
elif case == "no master ref":
- git(repo, "update-ref", "-d", f"refs/remotes/{master}")
+ git(repo, "update-ref", "-d", f"refs/remotes/{MASTER}")
result = subprocess.run(
- ["bash", "-e", "-c", list_step(engine)],
+ ["bash", "-e", "-c", list_step()],
cwd=repo,
env={
**os.environ,
diff --git a/.github/scripts/test_depot_backend_runs.py b/.github/scripts/test_depot_backend_runs.py
index d3acdbdde02a..182344dbf02a 100644
--- a/.github/scripts/test_depot_backend_runs.py
+++ b/.github/scripts/test_depot_backend_runs.py
@@ -7,7 +7,6 @@ from typing import Any
import pytest
-import ci_backend_relay
import depot_backend_runs as script
MASTER_SHA = "a" * 40
@@ -50,11 +49,6 @@ def test_gate_run_reports_the_gate_verdict(
assert run["head_sha"] == "abc"
-def test_names_the_workflow_and_gate_job_the_relay_reads() -> None:
- assert script.WORKFLOW_NAME == ci_backend_relay.DEPOT_WORKFLOW
- assert script.GATE_JOB_KEY == ci_backend_relay.GATE_JOB_KEY
-
-
@pytest.mark.parametrize(
"trigger,sha,kept",
[("push", MASTER_SHA, 1), ("push", BRANCH_SHA, 0), ("schedule", BRANCH_SHA, 1)],
diff --git a/.github/scripts/trunk-cross-lane.test.js b/.github/scripts/trunk-cross-lane.test.js
index 7ecb5f780215..a1766ed046de 100644
--- a/.github/scripts/trunk-cross-lane.test.js
+++ b/.github/scripts/trunk-cross-lane.test.js
@@ -38,7 +38,7 @@ for (const [name, files, mixed] of [
['prose beside Node', ['README.md', 'nodejs/src/ingestion/ingestion-consumer.ts'], false],
[
'a Python workflow beside a Rust workflow',
- ['.github/workflows/ci-backend.yml', '.github/workflows/ci-rust.yml'],
+ ['.depot/workflows/ci-backend.yml', '.github/workflows/ci-rust.yml'],
false,
],
]) {
diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.js
index 8010f8b915ae..be7435045b51 100644
--- a/.github/scripts/trunk-impacted-targets.js
+++ b/.github/scripts/trunk-impacted-targets.js
@@ -253,9 +253,10 @@ const TRIPWIRE_RULES = [
['.github/workflows/ci-nodejs.yml', NODE],
['.github/workflows/ci-nodejs-container.yml', NODE],
['.github/workflows/ci-mcp.yml', JAVASCRIPT],
- ['.github/workflows/ci-backend.yml', PYTHON],
+ ['.depot/workflows/ci-backend.yml', PYTHON],
['.github/workflows/ci-backend-update-test-timing.yml', PYTHON],
- ['.github/workflows/ci-backend-shadow-drift.yml', PYTHON],
+ ['.github/workflows/ci-backend-schema-mirror.yml', PYTHON],
+ ['.github/workflows/ci-depot-action-mirrors.yml', PYTHON],
['.github/workflows/ci-dagster.yml', PYTHON],
['.github/workflows/ci-rust.yml', RUST],
['.github/workflows/ci-rust-flags-integration.yml', RUST],
@@ -411,9 +412,6 @@ const TRIPWIRE_RULES = [
['.github/scripts/eng-analytics-weekly-digest.mjs', REPO_AUTOMATION],
['.github/scripts/ci-alerts-devex.js', REPO_AUTOMATION],
['.github/scripts/ci-alerts-devex.test.js', REPO_AUTOMATION],
- ['.github/scripts/ci_flake_overseer.py', REPO_AUTOMATION],
- ['.github/scripts/test_ci_flake_overseer.py', REPO_AUTOMATION],
- ['.github/scripts/compare-ci-runners.py', REPO_AUTOMATION],
['.github/scripts/report_workflow_run_traces.py', REPO_AUTOMATION],
['.github/scripts/test_report_workflow_run_traces.py', REPO_AUTOMATION],
['.github/auto-assign-labels.json', REPO_AUTOMATION],
@@ -589,6 +587,7 @@ const TRIPWIRE_RULES = [
['rust/.sqlx/**', RUST],
['hogli.yaml', UNIVERSAL],
['.github/**', UNIVERSAL],
+ ['.depot/workflows/**', UNIVERSAL],
['docker-compose*.yml', UNIVERSAL],
// Single-purpose images ahead of the fallback: each is read by exactly one
// workflow or suite, whose rule above already carries the radius.
@@ -931,16 +930,13 @@ function globToRegExp(glob) {
const TRIPWIRE_MATCHERS = TRIPWIRE_RULES.map(([glob, domain]) => [globToRegExp(glob), domain])
-// .depot/ holds Depot-runner shadows of the workflows and composite actions in
-// .github/, kept apples-to-apples with their canonicals by the shadow-drift
-// check, and their statuses are non-blocking. A shadow can only affect what
-// its canonical affects, so it resolves through the canonical's rules: the
-// ci-backend shadow takes the python lanes, and a shadow of something unplaced
-// still lands on the .github/** blanket.
-const DEPOT_MIRROR_PREFIX = '.depot/'
+// .depot/actions/ holds Depot CI's mirrors of the composite actions in .github/actions/,
+// kept in sync by the action-mirrors check, so a mirror resolves through its canonical
+// action's rules. Depot CI's own workflows under .depot/workflows/ carry rules of their own.
+const DEPOT_MIRROR_PREFIX = '.depot/actions/'
function canonicalPath(file) {
- return file.startsWith(DEPOT_MIRROR_PREFIX) ? `.github/${file.slice(DEPOT_MIRROR_PREFIX.length)}` : file
+ return file.startsWith(DEPOT_MIRROR_PREFIX) ? `.github/actions/${file.slice(DEPOT_MIRROR_PREFIX.length)}` : file
}
// The domain of the first rule matching the file, or null when no rule claims
diff --git a/.github/scripts/trunk-impacted-targets.test.js b/.github/scripts/trunk-impacted-targets.test.js
index b35ec963b4c7..a14e884d908a 100644
--- a/.github/scripts/trunk-impacted-targets.test.js
+++ b/.github/scripts/trunk-impacted-targets.test.js
@@ -135,7 +135,7 @@ test('a universal tripwire claims every known target', () => {
// included.
'docker-compose.base.yml',
// Trees that steer what every suite runs or what it runs against: the
- // Depot shadow of an action every workflow uses, the toolchain, the
+ // Depot mirror of an action every workflow uses, the toolchain, the
// service configs the stack mounts, and the markdownlint config every
// tree's prose obeys.
'.depot/actions/pnpm-install/action.yml',
@@ -525,7 +525,7 @@ test('the quarantine list claims the fullstack lanes and no rust crate', () => {
// paths-filter decides which jobs run inside a single run's own diff, so no
// run's outcome depends on it jointly with another queue entry. Its own CI and
-// its .depot shadow share the lane.
+// its .depot mirror share the lane.
test('the paths-filter action and its CI share the ci-tooling lane', () => {
for (const file of [
'.github/actions/paths-filter/src/filter.ts',
@@ -601,7 +601,7 @@ test('a single-language workflow claims that language rather than everything', (
)
}
for (const file of [
- '.github/workflows/ci-backend.yml',
+ '.depot/workflows/ci-backend.yml',
'.github/workflows/ci-python.yml',
'.github/workflows/ci-clickhouse-multinode-migrations.yml',
'.github/workflows/ci-clickhouse-util-udfs.yml',
@@ -781,20 +781,9 @@ test('the schema codegen scripts claim the product-surface lanes', () => {
)
})
-// A Depot shadow is kept apples-to-apples with its canonical by the
-// shadow-drift check and posts non-blocking statuses, so it can only affect
-// what the canonical affects. The narrowing direction is the guard: a shadow
-// of something no rule has placed must stay on the .github blanket rather
-// than fall through unclaimed.
-test('a depot shadow resolves through its canonical workflow rules', () => {
- assert.deepEqual(
- computeTargets(['.depot/workflows/ci-backend.yml'], CONTEXT),
- computeTargets(['.github/workflows/ci-backend.yml'], CONTEXT)
- )
- assert.deepEqual(
- computeTargets(['.depot/workflows/ci-backend-update-test-timing.yml'], CONTEXT),
- computeTargets(['mypy.ini'], CONTEXT)
- )
+// A Depot action mirror resolves through its canonical action, and a Depot workflow no rule
+// has placed stays on its own blanket rather than falling through unclaimed.
+test('depot mirrors and unplaced depot workflows resolve to everything', () => {
assert.deepEqual(computeTargets(['.depot/actions/pnpm-install/action.yml'], CONTEXT), EVERYTHING)
assert.deepEqual(computeTargets(['.depot/workflows/some-new-shadow.yml'], CONTEXT), EVERYTHING)
// Shadow markdown is prose like any other.
diff --git a/.github/scripts/turbo-discover.test.js b/.github/scripts/turbo-discover.test.js
index 62faf1f68135..02a2278de10f 100644
--- a/.github/scripts/turbo-discover.test.js
+++ b/.github/scripts/turbo-discover.test.js
@@ -3,8 +3,7 @@
// Pins turbo-discover's DJANGO_SEGMENTS table to the Django pytest invocations
// in ci-backend.yml. The table sizes the shards each segment gets, so a segment
// listing paths the workflow no longer runs (or missing ones it does) budgets
-// wall time for a run that never happens. Both workflow copies are read: the
-// Depot mirror runs the same matrix and drifts on its own.
+// wall time for a run that never happens.
//
// Reads the workflow rather than a fixture on purpose — the workflow is the one
// side that can drift, and there is nothing else to compare the table against.
@@ -19,7 +18,7 @@ const path = require('node:path')
const { DJANGO_SEGMENTS, getIsolatedProducts, loadJsonTargets } = require('./turbo-discover')
const REPO_ROOT = path.join(__dirname, '..', '..')
-const WORKFLOWS = ['.github/workflows/ci-backend.yml', '.depot/workflows/ci-backend.yml']
+const WORKFLOW = '.depot/workflows/ci-backend.yml'
// pytest takes `./posthog/queries/` and `posthog` for the same directory, and
// --ignore drops the trailing slash. DJANGO_SEGMENTS stores prefixes, so every
@@ -68,21 +67,19 @@ function parseTemporalTargets(text) {
return invocation[1].split(/\s+/)
}
-for (const workflow of WORKFLOWS) {
- const text = fs.readFileSync(path.join(REPO_ROOT, workflow), 'utf8')
- const core = parseCoreStep(text)
- const expected = {
- Core: { include: core.core, exclude: core.ignores },
- CorePOE: { include: core.poe, exclude: [...core.ignores, ...core.poeIgnores] },
- Temporal: { include: parseTemporalTargets(text), exclude: [] },
- }
+const text = fs.readFileSync(path.join(REPO_ROOT, WORKFLOW), 'utf8')
+const core = parseCoreStep(text)
+const expected = {
+ Core: { include: core.core, exclude: core.ignores },
+ CorePOE: { include: core.poe, exclude: [...core.ignores, ...core.poeIgnores] },
+ Temporal: { include: parseTemporalTargets(text), exclude: [] },
+}
- for (const [segment, paths] of Object.entries(expected)) {
- test(`${segment} segment matches the pytest targets in ${workflow}`, () => {
- assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].include), sorted(paths.include.map(toPrefix)))
- assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].exclude), sorted(paths.exclude.map(toPrefix)))
- })
- }
+for (const [segment, paths] of Object.entries(expected)) {
+ test(`${segment} segment matches the pytest targets in ${WORKFLOW}`, () => {
+ assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].include), sorted(paths.include.map(toPrefix)))
+ assert.deepEqual(sorted(DJANGO_SEGMENTS[segment].exclude), sorted(paths.exclude.map(toPrefix)))
+ })
}
// The backend test selector is the third copy of the partition: it routes each
diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
deleted file mode 100644
index 2439c30e0a98..000000000000
--- a/.github/workflows/ci-backend.yml
+++ /dev/null
@@ -1,4661 +0,0 @@
-# This workflow runs all of our backend django tests.
-#
-# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
-# .github/workflows/ci-backend-update-test-timing.yml action
-name: Backend CI
-on:
- # No push or schedule trigger: the master push and the hourly full-matrix master run are on
- # Depot CI, from .depot/workflows/ci-backend.yml.
- workflow_dispatch:
- inputs:
- clickhouseServerVersion:
- description: ClickHouse server version. Leave blank for default
- type: string
- pull_request:
- # PRs run the snob-selected Django subset (one shard per active segment)
- # for fast feedback, draft or ready. When selection can't be trusted, a
- # ready PR falls back to the full matrices and a draft skips them instead,
- # deferring to its ready run. turbo-tests (product tests) still skip drafts.
- # On a legacy diff the product matrix is narrowed the same way: the products the
- # selector reached plus the ones Turbo saw change (see turbo-discover).
- # The merge queue's trunk-merge/** run and the hourly scheduled master run
- # always run the full matrices; the queue run is what actually gates master.
- # Master pushes run only the per-commit checks (migrations + schema cache,
- # OpenAPI types, repo checks). To force the full matrices on a PR, add the
- # `run-ci-backend` label. Cheap checks still run on drafts.
- # The `no-ci` label does the opposite: it silences this workflow on a draft
- # entirely (prototypes/spikes).
- # No labeled/unlabeled triggers: GitHub cannot filter a label trigger by name,
- # so every unrelated label re-ran these matrices against a commit CI had
- # already covered. A label here takes effect from the next push, or from
- # ready_for_review.
- # The events_json leg runs wherever the legacy tests run, and on every third
- # hourly master run: the paths in .github/new-events-schema-targets.txt rerun against the
- # native-JSON events table, all of them on full runs and the reached ones on
- # narrowed runs. A PR whose product-only change forced the full run reruns only
- # that product's paths, because the merge queue runs the whole list before merge.
- # The `test-new-events-schema` label opts a PR into the doubled
- # matrices that run the whole suite in both events schemas, in place of that leg.
- # Apply it before a push or ready_for_review because label events do not start
- # this workflow. Manual dispatches also run both modes.
- types: [opened, synchronize, reopened, ready_for_review]
-
-concurrency:
- # PRs: one active run per branch, cancel stale. Push: per-SHA so master
- # pushes never cancel each other (check-migrations always completes).
- group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.head_ref || github.ref }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-env:
- # Schema cache epoch. Bump to abandon every shared schema cache entry at once
- # (key is posthog-schema-mig-<epoch>-<hash>). Single-sourced so it can't drift
- # between the merge-base (PR) and HEAD (push) key computations below.
- # ci-e2e-playwright.yml, ci-dagster.yml, ci-mcp.yml and ci-rust-flags-integration.yml restore by the same key; keep their copies in sync when bumping.
- SCHEMA_CACHE_EPOCH: v2
- # Hypothesis constants cache epoch. Bump to abandon every shared entry at once
- # (key is posthog-hypothesis-constants-<version>-<epoch>-<isoyear>-<isoweek>); used by
- # the Django test shards below.
- HYPOTHESIS_CONSTANTS_EPOCH: v1
- SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
- DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
- REDIS_URL: 'redis://localhost'
- CLICKHOUSE_HOST: 'localhost'
- CLICKHOUSE_SECURE: 'False'
- CLICKHOUSE_VERIFY: 'False'
- # Restricted `autoresearch` user defined in docker/clickhouse/users-dev.xml,
- # pointed at the CI test DB (posthog_test, not `default` as in regular dev).
- CLICKHOUSE_TEST_CLUSTER_HOST: 'localhost'
- CLICKHOUSE_TEST_CLUSTER_DATABASE: 'posthog_test'
- CLICKHOUSE_TEST_CLUSTER_USER: 'autoresearch'
- CLICKHOUSE_TEST_CLUSTER_PASSWORD: 'autoresearchpass'
- CLICKHOUSE_TEST_CLUSTER_SECURE: 'False'
- CLICKHOUSE_TEST_CLUSTER_VERIFY: 'False'
- TEST: 1
- # The root conftest ends a green pytest session with os._exit once every report is written.
- # The hourly run on Depot CI turns this off, because it also hides a crash during interpreter teardown.
- POSTHOG_PYTEST_HARD_EXIT: '1'
- CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}
- CLICKHOUSE_COMPAT_PYTEST_TARGETS: 'posthog/clickhouse ee/clickhouse'
- OBJECT_STORAGE_ENABLED: 'True'
- OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
- OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
- OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
- UV_HTTP_TIMEOUT: 120
- # A 4 GiB thread stack cache avoids the uv exit segfault (code 139) on glibc before 2.43.
- # Remove when uv ships the fix: https://github.com/astral-sh/uv/issues/21428
- GLIBC_TUNABLES: glibc.pthread.stack_cache_size=0x100000000
- # tests would intermittently fail in GH actions
- # with exit code 134 _after passing_ all tests
- # this appears to fix it
- # absolute wild tbh https://stackoverflow.com/a/75503402
- DISPLAY: ':99.0'
- # this is a fake key so this workflow can run for external contributors as they do not have access to secrets (that we don't need here)
- OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }}
- RUNS_ON_INTERNAL_PR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}
- SANDBOX_JWT_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }}
-
-permissions:
- contents: read
- actions: read
- pull-requests: write
-
-jobs:
- # Learning mode: nothing reads these outputs, so a recommendation cannot change what runs.
- # Fork and Dependabot runs get no secret; the queue always runs everything anyway.
- # The step fails open, because this telemetry must not turn a passing workflow red.
- dynamic-ci-filter:
- name: Trunk Dynamic CI (Learning Mode)
- if: >-
- github.event_name == 'pull_request' &&
- github.repository == 'PostHog/posthog' &&
- github.event.pull_request.head.repo.full_name == github.repository &&
- github.actor != 'dependabot[bot]' &&
- !startsWith(github.head_ref, 'trunk-merge/') &&
- (github.event.pull_request.draft != true ||
- !contains(github.event.pull_request.labels.*.name, 'no-ci'))
- runs-on: ubuntu-24.04
- timeout-minutes: 5
- permissions: {}
- steps:
- - name: Ask Trunk which jobs this diff needs
- continue-on-error: true
- uses: trunk-io/dynamic-ci@7e3af9331e8ebdfe0c71ba7d0ff6b7424dde8c57 # v1
- with:
- token: ${{ secrets.TRUNK_API_TOKEN }}
-
- # Job to decide if we should run backend ci
- # See .github/actions/paths-filter/README.md for filter semantics
- changes:
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 5
- # Depot CI runs every pull request event from .depot/workflows/ci-backend.yml, and
- # its gate posts the required check. Every job that needs this one skips on a pull
- # request, and the gate treats skipped as success.
- if: github.event_name != 'pull_request'
- name: Determine need to run backend and migration checks
- # Set job outputs to values from filter step
- outputs:
- # run-ci-backend is documented as forcing the full matrices, but every
- # gate reads this output, so the label cannot reach them while the paths
- # filter says false. Nothing under .github/ is in the backend filter, so
- # a sharding or timing script change was untestable on its own PR.
- backend: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && 'true' || (steps.filter.outputs.backend || 'true') }}
- backend_files: ${{ steps.filter.outputs.backend_files }}
- migrations: ${{ steps.filter.outputs.migrations || 'true' }}
- # Added and modified files only. A squash retirement deletes thousands of
- # migrations, and a step env string over 128 KiB fails the spawn (E2BIG).
- migrations_review_files: ${{ steps.filter.outputs.migrations_review_files }}
- migrations_any: ${{ steps.filter.outputs.migrations_any || 'true' }}
- # What the replay gate in check-migrations keys off. A ClickHouse or
- # async-migration change does not pay for a replay of a history the Django
- # planner never sees.
- migration_replay: ${{ steps.filter.outputs.migration_replay || 'true' }}
- persons_sql: ${{ steps.filter.outputs.persons_sql || 'true' }}
- tasks_temporal: ${{ steps.filter.outputs.tasks_temporal || 'true' }}
- openapi_types: ${{ steps.filter.outputs.openapi_types || 'true' }}
- # The label has to reach discovery as well: `backend` alone only starts the
- # discover job, which then sees no legacy change and no affected product and
- # emits an empty matrix. A forced legacy change makes it test everything.
- legacy: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && 'true' || (steps.filter.outputs.legacy || 'true') }}
- schema: ${{ steps.filter.outputs.schema || 'true' }}
- product_yamls: ${{ steps.filter.outputs.product_yamls || 'false' }}
- repo_invariants: ${{ steps.filter.outputs.repo_invariants || 'false' }}
- sdk_manifests: ${{ steps.filter.outputs.sdk_manifests || 'true' }}
- product_yamls_files: ${{ steps.filter.outputs.product_yamls_files }}
- timing_scripts: ${{ steps.filter.outputs.timing_scripts }}
- steps:
- # For pull requests it's not necessary to checkout the code, but we
- # also want this to run on master so we need to checkout
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- clean: false
- sparse-checkout: |
- .github/actions/paths-filter
- sparse-checkout-cone-mode: false
- - name: Clean up data directories with container permissions
- run: |
- # Use docker to clean up files created by containers
- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
- continue-on-error: true
-
- - uses: ./.github/actions/paths-filter
- # Skipped on master push: every output then defaults to true.
- if: github.event_name != 'push'
- id: filter
- with:
- token: ${{ github.token }}
- list-files: 'escape'
- filters: |
- backend:
- # Avoid running backend tests for irrelevant changes
- # NOTE: we are at risk of missing a dependency here. We could make
- # the dependencies more clear if we separated the backend/frontend
- # code completely
- - 'ee/**/*'
- - '!ee/frontend/**' # frontend code under ee/ must not trigger backend
- - 'common/__init__.py'
- - 'common/hogql_parser/**'
- - 'rust/hogql/parser/**'
- - 'common/hogvm/**'
- - 'common/ingestion/**'
- - 'common/migration_utils/**'
- - 'common/plugin_transpiler/**'
- # The personhog gRPC stubs that posthog/personhog_client imports
- - 'packages/personhog-proto/**'
- - 'posthog/**/*'
- - 'products/**/backend/**/*'
- # Python outside backend/ — products like logs and
- # posthog_ai point `backend:test` at extra roots
- # (skills/, scripts/). Catch any .py change so the
- # backend job triggers when only those files move.
- - 'products/**/*.py'
- # products/desktop is the imported desktop app; it has its own
- # desktop-* CI and must not drag this suite onto its PRs
- - '!products/desktop/**'
- # Each product's package.json declares the per-product
- # backend:test script that turbo-discover reads to
- # compose the Product matrix. Without this entry,
- # widening or fixing the script doesn't trigger
- # tests it controls.
- - 'products/*/package.json'
- # Tests read these MCP manifests from disk. Each one is a turbo input of
- # its reader (tasks directly, posthog_ai through feature_flags'
- # backend:mcp-surface task). Add a manifest here only with that wiring.
- - products/feature_flags/mcp/tools.yaml
- - products/tasks/mcp/tools.yaml
- - 'products/canvas/packages/canvas_builder/**'
- - bin/build-schema-latest-versions.py
- - bin/check_uv_python_compatibility.py
- - bin/find_python_dependencies.py
- - pyproject.toml
- - uv.lock
- - requirements.txt
- - requirements-dev.txt
- - mypy.ini
- - pytest.ini
- - conftest.py # Root conftest loads for every pytest run
- - .test_quarantine.json # Un-quarantining must re-run the affected tests
- # tach.toml's exclude list shapes the import map turbo-discover
- # reads for the dependent cascade, and `tach check` validates
- # the declarations — a graph-only PR must still dispatch both.
- - tach.toml
- # generates posthog/schema.py
- - frontend/src/queries/schema.json
- # Operator parity check
- - rust/feature-flags/src/properties/property_models.rs
- - frontend/src/products.json # Loaded at runtime by posthog/products.py
- - 'products/*/manifest.tsx' # Generates products.json
- # hogli setup commands this workflow runs: db:restore-test-db
- # and product:bootstrap/lint. Scoped to the modules those
- # commands actually import — framework, manifest, boot
- # modules, db_schema, product — so unrelated hogli commands
- # (doctor, devbox, ...) don't trigger this suite.
- - 'tools/hogli/**'
- - hogli.yaml
- - 'tools/hogli-commands/hogli_commands/db_schema.py'
- - 'tools/hogli-commands/hogli_commands/prechecks.py'
- - 'tools/hogli-commands/hogli_commands/telemetry_props.py'
- - 'tools/hogli-commands/hogli_commands/hint_hook.py'
- - 'tools/hogli-commands/hogli_commands/hints.py'
- - 'tools/hogli-commands/hogli_commands/product/**'
- - 'tools/hogli-commands/hogli_commands/product_structure.yaml'
- # owners_yaml is a runtime dependency of product code (stamphog reads
- # owners.yaml through it), so a resolver change has to re-run its
- # dependents; turbo-discover finds the products that import it.
- # Deliberately not in `legacy`, because a leaf-package change should
- # run its dependents rather than the whole Django suite.
- - 'packages/owners-yaml/**'
- # Transitional: branches that predate the move still carry the
- # resolver at tools/owners.
- - 'tools/owners/**'
- # product:lint --all checks these generated files against the tree,
- # so a baseline-only edit must run it too. The crossings baseline is
- # also what product:lint reads to let a wiring location leave a
- # product's turbo.json inputs, so both edits must run the checks.
- - products/isolation_baseline.txt
- - products/model_crossing_uses_baseline.txt
- - 'products/*/turbo.json'
- # Make sure we run if someone is explicitly changing the workflow
- - .github/workflows/ci-backend.yml
- - .github/new-events-schema-targets.txt
- - .github/actions/setup-uv/**
- - .github/clickhouse-versions.json
- # We use docker compose for tests, make sure we rerun on
- # changes to docker-compose.dev.yml e.g. dependency
- # version changes
- - docker-compose.dev.yml
- - docker-compose.profiles.yml
- - docker-compose.base.yml
- - bin/wait-for-docker
- - bin/ci-wait-for-docker
- - frontend/public/email/*
- - 'docker/clickhouse/**'
- # Markdown-only changes (e.g. a README under posthog/) match the
- # broad globs above but exercise no backend code — exclude them so
- # a docs-only PR doesn't run the whole backend suite.
- # These excludes are load-bearing: an over-broad exclude silently
- # skips the suite rather than over-running it, so only exclude
- # patterns that truly never affect backend. They also drop matched
- # files from `backend_files`.
- - '!**/*.md'
- - '!**/*.mdx'
- legacy:
- # Non-product backend code — when only products/ change,
- # turbo-discover uses Turbo query affectedness to detect changed
- # products and decide whether Django runs.
- # Everything from backend: EXCEPT products/**/backend/**/*
- - 'ee/**/*'
- - '!ee/frontend/**' # frontend code under ee/ must not trigger backend
- - 'common/__init__.py'
- - 'common/hogql_parser/**'
- - 'rust/hogql/parser/**'
- - 'common/hogvm/**'
- - 'common/ingestion/**'
- - 'common/migration_utils/**'
- - 'common/plugin_transpiler/**'
- # The personhog gRPC stubs that posthog/personhog_client imports
- - 'packages/personhog-proto/**'
- - 'posthog/**/*'
- - bin/build-schema-latest-versions.py
- - bin/check_uv_python_compatibility.py
- - bin/find_python_dependencies.py
- - pyproject.toml
- - uv.lock
- - requirements.txt
- - requirements-dev.txt
- - mypy.ini
- - pytest.ini
- - conftest.py # Root conftest loads for every pytest run
- - .test_quarantine.json
- - frontend/src/products.json
- - 'products/*/manifest.tsx'
- - rust/feature-flags/src/properties/property_models.rs
- - .github/workflows/ci-backend.yml
- - .github/new-events-schema-targets.txt
- - .github/actions/setup-uv/**
- - .github/clickhouse-versions.json
- - docker-compose.dev.yml
- - docker-compose.profiles.yml
- - docker-compose.base.yml
- - bin/wait-for-docker
- - bin/ci-wait-for-docker
- - frontend/public/email/*
- - 'docker/clickhouse/**'
- # Same rationale as the backend filter's excludes above: docs-only
- # changes match the broad globs but exercise no legacy backend code,
- # so they shouldn't trigger the Django/turbo-discover path either.
- - '!**/*.md'
- - '!**/*.mdx'
- # Generated from frontend/src/queries/schema.json (see the `schema`
- # filter below). Excluding them here routes schema-only PRs into
- # turbo-discover's schema-impact narrowing instead of the
- # all-products legacy path. This cannot skip Django: the
- # schemaChanged branch hard-sets run_legacy=true. It also cannot
- # hide a hand-edit to the generated files — ci-python.yml
- # regenerates schema.py and fails on any diff, and both files stay
- # in the `schema` (and `backend`) filters so turbo-discover still
- # takes the schema path for them.
- - '!posthog/schema.py'
- - '!posthog/schema_enums.py'
- schema:
- # Tracked separately from `legacy` so turbo-discover can
- # diff schema.json and narrow the product matrix to
- # products that import the changed types.
- - frontend/src/queries/schema.json
- # The generated outputs are excluded from `legacy` above; listing
- # them here keeps run_legacy=true (full Django) for any schema
- # change, including a stray hand-edit to a generated file.
- - posthog/schema.py
- - posthog/schema_enums.py
- migrations:
- - 'docker/clickhouse/**'
- - 'posthog/migrations/*.py'
- - 'products/*/backend/migrations/*.py'
- - 'products/*/migrations/*.py' # Legacy structure
- migrations_review:
- - added|modified: 'posthog/migrations/*.py'
- - added|modified: 'products/*/backend/migrations/*.py'
- - added|modified: 'products/*/migrations/*.py'
- # Any migration file in any app (broader than `migrations` above,
- # which omits ee/ and posthog sub-package dirs). Gates the
- # deleted-migration check in repo-checks.
- migrations_any:
- - '**/migrations/*.py'
- # Django migration files in any app, plus the routing files that decide
- # which app labels land in which database. Both change the plan a
- # from-scratch migrate builds. Match everything and exclude the two
- # non-Django directories, so a new app is covered without an edit here.
- # An app missed by a positive list would leave the replay gate silently
- # off for it.
- migration_replay:
- - '**/migrations/*.py'
- - '!posthog/clickhouse/migrations/*.py'
- - '!posthog/async_migrations/migrations/*.py'
- - 'products/db_routing.yaml'
- - 'posthog/product_db_config.py'
- - 'posthog/product_db_router.py'
- persons_sql:
- - 'rust/persons_migrations/*.sql'
- - 'rust/bin/migrate-persons'
- timing_scripts:
- - '.github/scripts/optimize_test_durations.py'
- - '.github/scripts/turbo-discover.js'
- tasks_temporal:
- - 'products/tasks/backend/temporal/**/*'
- repo_invariants:
- # posthog/test/repo_invariants/ reads AGENTS.md and the semgrep
- # rule ids it names. Both are excluded from the backend filter
- # (markdown is dropped outright, .semgrep is not listed), so a
- # change to either would skip repo-checks and report green while
- # leaving a tag pointing at a rule that no longer exists.
- # The out-of-band domain ratchet reads these trees, so a resolver added there must run it.
- - 'nodejs/src/**'
- - 'rust/**'
- - 'products/desktop/**'
- - 'AGENTS.md'
- - '.semgrep/**'
- - 'pyproject.toml'
- # A generated file in a frontend-only PR still needs a known producer.
- - '**/*.generated.*'
- - '**/generated/**'
- # test_generated_files_are_registered.py also reads the projection
- # registry and the ci-python paths filter it must stay in step with.
- - 'tools/hogli-commands/hogli_commands/projections.py'
- - '.github/workflows/ci-python.yml'
- - 'products/ruff.toml'
- - '.github/scripts/**'
- # test_signals_scout_tool_citations.py checks the MCP tool names that the
- # canonical scout skills cite against the committed tool catalog. The backend
- # filter matches only the .py files in a skill and nothing under services/mcp/.
- # The same test fails when a file it reads has no match in this filter.
- - 'products/signals/skills/**'
- - 'services/mcp/schema/tool-definitions.json'
- - 'services/mcp/schema/generated-tool-definitions.json'
- # _EXTERNAL_COMPANION_SKILL_DIRS in products/signals/backend/scout_harness/lazy_seed.py
- # names this companion skill from another product.
- - 'products/replay_vision/skills/exploring-replay-vision-observations/**'
- sdk_manifests:
- # Everything the SDK major-version guard reads. Lockfiles are listed
- # because a floor-only or `*` spec can move to a new major through
- # its lock alone.
- - '**/package.json'
- - 'pnpm-workspace.yaml'
- - 'pnpm-lock.yaml'
- - 'products/desktop/pnpm-workspace.yaml'
- - 'products/desktop/pnpm-lock.yaml'
- - 'pyproject.toml'
- - 'uv.lock'
- - 'services/llm-gateway/pyproject.toml'
- - 'services/llm-gateway/uv.lock'
- - 'common/ingestion/acceptance_tests/requirements.txt'
- - 'tools/infra-scripts/clitools/requirements.txt'
- - 'tools/infra-scripts/clitools/constraints.txt'
- - 'rust/Cargo.toml'
- - 'rust/Cargo.lock'
- - 'cli/Cargo.toml'
- - 'cli/Cargo.lock'
- - 'bin/hobby-installer/go.mod'
- - 'bin/hobby-installer/go.sum'
- - '.github/scripts/check_sdk_majors.py'
- # The guard scans every package.json for an SDK declaration.
- - '**/package.json'
- - 'rust/Cargo.toml'
- - 'cli/Cargo.toml'
- - 'bin/hobby-installer/go.mod'
- product_yamls:
- # Validates owner team slugs against PostHog/posthog collaborator teams,
- # and lints the distributed owners.yaml files (schema, conflicts, dead
- # globs, coverage). Costs one GitHub API call; gated here so we only pay
- # it when ownership actually changes. Drives the validate-product-yamls job.
- - 'products/*/product.yaml'
- - '**/owners.yaml'
- # Resolver changes must re-run the owners.yaml lint too.
- - 'packages/owners-yaml/**'
- # Transitional: branches that predate the move still carry the
- # resolver at tools/owners.
- - 'tools/owners/**'
- openapi_types:
- # Generated OpenAPI types - validate they match schema
- - 'frontend/src/generated/**/*'
- - 'frontend/src/lib/agentScopes.generated.ts'
- - 'products/*/frontend/generated/**/*'
- - 'services/mcp/src/generated/**/*'
- - 'services/mcp/src/api/generated.ts'
- - 'tools/openapi-codegen/**/*'
- - 'services/mcp/scripts/lib/**'
- - 'frontend/bin/generate-openapi-types.mjs'
- - 'frontend/src/lib/api-orval-mutator.ts'
- - 'services/mcp/scripts/**/*'
- # MCP tool config sources - feed generate-tools.ts
- - 'services/mcp/definitions/**/*.yaml'
- - 'services/mcp/definitions/prompts/**/*.md'
- - 'products/*/mcp/**/*.yaml'
- - 'products/*/mcp/prompts/**/*.md'
- - 'services/mcp/schema/tool-definitions.json'
- - 'services/mcp/schema/tool-definitions-v2.json'
- - 'services/mcp/src/tools/generated/**/*'
- - 'services/mcp/schema/generated-tool-definitions.json'
- - 'services/mcp/schema/tool-definitions-all.json'
- # Exec command reference templates - feed generate-exec-docs.ts
- - 'services/mcp/src/templates/sections/**/*'
- - 'services/mcp/schema/exec-command-reference.md'
- # Desktop API client - regenerated by bin/build-desktop-api-client.sh
- - 'products/desktop/packages/api-client/src/generated.ts'
- - 'products/desktop/packages/api-client/endpoint-allowlist.json'
- # Pins typed-openapi exactly, so a generator bump changes this file and not only the desktop lockfile.
- - 'products/desktop/packages/api-client/package.json'
- - 'products/desktop/packages/api-client/scripts/**'
- - 'bin/build-desktop-api-client.sh'
-
- detect-snapshot-mode:
- name: Detect snapshot mode
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 5
- needs: [changes]
- # Master pushes skip the matrices (see the trigger comment); skipping this
- # root cascades to every downstream test job via their result checks.
- if: needs.changes.outputs.backend == 'true' && github.event_name != 'push'
- outputs:
- mode: ${{ steps.detect.outputs.mode }}
- steps:
- - name: Detect mode
- id: detect
- env:
- PR_REPO: ${{ github.event.pull_request.head.repo.full_name }}
- REPO: ${{ github.repository }}
- HAS_NO_SNAPSHOT_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'no-snapshot-update') }}
- AUTHOR: ${{ github.actor }}
- run: |
- if [ "$PR_REPO" != "$REPO" ] && [ -n "$PR_REPO" ]; then
- echo "mode=check" >> $GITHUB_OUTPUT
- echo "Fork detected - running in CHECK mode (no commits allowed)"
- elif [ "$HAS_NO_SNAPSHOT_LABEL" == "true" ]; then
- echo "mode=check" >> $GITHUB_OUTPUT
- echo "::notice::🔍 Running in CHECK mode - 'no-snapshot-update' label detected"
- else
- echo "Workflow triggered by: $AUTHOR"
- # Dependabot is excluded - it creates new PRs that may need snapshot updates
- # Other bots (github-actions, posthog-bot) commit snapshots and must use CHECK mode to avoid loops
- if [[ "$AUTHOR" != "dependabot[bot]" ]] && \
- ([[ "$AUTHOR" == *"github-actions"* ]] || [[ "$AUTHOR" == *"[bot]"* ]] || [[ "$AUTHOR" == "posthog-bot" ]]); then
- echo "mode=check" >> $GITHUB_OUTPUT
- echo "::notice::🔍 Running in CHECK mode - snapshots must match exactly"
- else
- echo "mode=update" >> $GITHUB_OUTPUT
- echo "::notice::🔄 Running in UPDATE mode - snapshots can be updated"
- fi
- fi
-
- # Fast pre-job: determines which products need testing and if Django should run
- # Only needs pnpm + node — no Python, Docker, or services
- # Runs on depot to match the product-test runner environment
- turbo-discover:
- needs: changes
- # Skipped on master push: the merge queue already ran the full matrices for
- # every landed commit, and the hourly scheduled run keeps master coverage.
- if: needs.changes.outputs.backend == 'true' && github.event_name != 'push'
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 20
- name: Discover product tests
- env:
- # Whether this run selects backend tests at all: a PR, off the merge queue's
- # trunk-merge/** run (that run is the merge gate and stays full), and without
- # the run-ci-backend label. turbo-discover.js reads it too, so a run that never
- # selects reports an empty mode rather than a selector error.
- SELECTION_APPLIES: ${{ github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') && !contains(github.event.pull_request.labels.*.name, 'run-ci-backend') && needs.changes.outputs.backend == 'true' }}
- # GitHub tests PRs as synthetic merges. The first parent pins the exact integrated
- # base for ordinary, stacked, and merge-queue PRs.
- TURBO_SCM_BASE: ${{ github.event_name == 'pull_request' && format('{0}^1', github.sha) || '' }}
- outputs:
- run_legacy: ${{ steps.discover.outputs.run_legacy }}
- run_legacy_reason: ${{ steps.discover.outputs.run_legacy_reason }}
- matrix: ${{ steps.discover.outputs.matrix }}
- schema_cache_key: ${{ steps.schema-key.outputs.key }}
- schema_migrations_key: ${{ steps.schema-key.outputs.migrations_key }}
- schema_prefix_key: ${{ steps.schema-key.outputs.prefix_key }}
- django_shards: ${{ steps.discover.outputs.django_shards }}
- # Which Django tests this PR runs. Empty mode = this run does not select.
- mode: ${{ steps.discover.outputs.mode }}
- # The rest of that decision (segment file lists, run_poe, run_temporal,
- # segment_shards) as one JSON object; consumers read it with fromJSON.
- selection: ${{ steps.discover.outputs.selection }}
- # The posthog-ci-test-selection event properties, as one JSON object.
- telemetry: ${{ steps.discover.outputs.telemetry }}
-
- steps:
- # A deep checkout is slow with or without blob:none: the blobless one fetches every file of
- # the tree in a second request. Check out at depth 1, then add the history the merge-base
- # steps read, without file contents.
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
- - name: Fetch history without file contents
- # --depth, not --deepen: --deepen adds no history once GitHub has replaced the PR's
- # merge ref, because no ref then points at GITHUB_SHA.
- run: |
- git fetch --no-tags --filter=blob:none --depth=1000 origin "$GITHUB_SHA"
- echo "History depth: $(git rev-list --count HEAD)"
-
- - name: Verify PR merge for test selection
- id: verify-merge
- if: github.event_name == 'pull_request'
- env:
- PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- run: |
- set -euo pipefail
- trap 'echo "::error::Cannot verify the PR merge checkout. Check the checkout ref and the history fetch before retrying." >&2' ERR
- test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
- test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3
- test "$(git rev-parse HEAD^2)" = "$PR_HEAD_SHA"
- git rev-parse --verify "$TURBO_SCM_BASE^{commit}"
-
- - name: Fetch PR base for schema cache
- if: github.event_name == 'pull_request'
- env:
- BASE_REF: ${{ github.event.pull_request.base.ref }}
- run: git fetch --no-tags --depth=1000 --filter=blob:none origin "$BASE_REF:refs/remotes/origin/$BASE_REF"
-
- - name: Compute schema cache key from merge-base
- id: schema-key
- if: github.event_name == 'pull_request'
- env:
- BASE_REF: ${{ github.event.pull_request.base.ref }}
- run: |
- # HEAD is the synthetic merge commit; HEAD^2 is the PR branch tip.
- # The history fetch and the base-ref fetch above ensure the
- # full ancestry needed to find the divergence point is available.
- MERGE_BASE=$(git merge-base HEAD^2 "origin/${BASE_REF}" 2>/dev/null || echo "")
- if [ -z "$MERGE_BASE" ]; then
- echo "key=" >> $GITHUB_OUTPUT
- echo "migrations_key=" >> $GITHUB_OUTPUT
- echo "::notice::merge-base not found (branch too stale?) — schema cache will be skipped"
- exit 0
- fi
- # Routing decides which app labels reach the dump, and the consumers replay them
- # from the checked-out copy, so these inputs belong in the key below. A PR that
- # edits any of them matches no master dump, the merge-base one included: drop
- # both keys and let migrations run from scratch.
- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)
- DB_ROUTING_HEAD=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)
- if [ "$DB_ROUTING" != "$DB_ROUTING_HEAD" ]; then
- echo "key=" >> $GITHUB_OUTPUT
- echo "migrations_key=" >> $GITHUB_OUTPUT
- echo "::notice::db routing config or router differs from the merge base, so the schema cache is skipped"
- exit 0
- fi
- # Exact branch-point key, kept as the fallback restore key below.
- echo "key=posthog-schema-master-${MERGE_BASE}" >> $GITHUB_OUTPUT
- # Last-resort restore prefix: matches the newest saved dump of any
- # migration set, for the window where both exact keys miss. Gated on
- # checkout age (a day) because a re-run of an old run keeps its original
- # commit: a stale checkout can lack migrations the newest dump records,
- # and the migrate top-ups only apply forward, so they cannot repair a
- # dump that is newer than the code.
- HEAD_AGE_SECONDS=$(( $(date +%s) - $(git show -s --format=%ct HEAD) ))
- if [ "$HEAD_AGE_SECONDS" -lt 86400 ]; then
- echo "prefix_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-" >> $GITHUB_OUTPUT
- fi
- # Shared key: hash merge-base schema inputs, not PR head, so PR migrations still
- # apply as the pytest delta. uv.lock covers third-party migration changes; the
- # postgres image is folded in so a server-version bump invalidates the key.
- # Non-Django migration frameworks are excluded from this Postgres schema dump.
- # SCHEMA_CACHE_EPOCH (workflow env) prefixes the key as a manual cache-bust knob.
- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" \
- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \
- | grep -vE '/(clickhouse|async_migrations)/migrations/' \
- | LC_ALL=C sort) || true
- PG_IMAGE=$(git show "${MERGE_BASE}:docker-compose.base.yml" 2>/dev/null \
- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')
- if [ -z "$MIG_FILES" ]; then
- echo "migrations_key=" >> $GITHUB_OUTPUT
- echo "::notice::no migration files matched — restore will fall back to the SHA schema key"
- else
- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)
- echo "migrations_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT
- fi
-
- # Dedicated app: every "Mint setup-action GitHub token" step in this file
- # draws on posthog-devex-general rather than the posthog-setup-actions app
- # the other workflows share. Backend CI is the largest single consumer of
- # setup-action calls in the repo, enough to threaten a shared 15k/hr bucket
- # at merge peaks, so it gets one to itself.
- - name: Mint setup-action GitHub token
- id: setup-gh-token
- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}
- skip-token-revoke: true
-
- # turbo-discover.js needs Node and the turbo binary, not the installed workspace.
- - name: Set up pnpm and Node
- uses: ./.github/actions/setup-pnpm
- with:
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- # Needs pnpm from the step above.
- - name: Install turbo
- uses: ./.github/actions/setup-turbo
-
- # turbo-discover.js runs .github/scripts/tach_map.py under `uv run
- # --no-project` for the dependent cascade; the script pins tach, so no
- # Python project sync is needed. A missing uv widens the cascade to
- # every product rather than failing the job.
- - uses: ./.github/actions/setup-uv
- with:
- enable-cache: false
- continue-on-error: true
-
- - name: Restore test durations from cache
- # Shard counts are derived from .test_durations (Amdahl). Restore the newest
- # cache (refreshed by ci-backend-update-test-timing.yml several times a day);
- # the fallback step below covers a miss.
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: .test_durations
- # run_id never matches — forces the prefix restore-key to the newest entry.
- key: posthog-test-durations-${{ github.run_id }}
- restore-keys: |
- posthog-test-durations-
-
- - name: Fall back to the last published durations
- # .test_durations is not in the repo. The timing workflow refreshes the
- # cache several times a day and publishes the same files as an artifact
- # of its own runs. On a cache miss (retention lapsed, timing workflow
- # down for two weeks) take the newest artifact; if that is gone too,
- # pytest-split splits by count and turbo-discover sizes from file counts
- # until the next timing run.
- if: ${{ hashFiles('.test_durations') == '' }}
- continue-on-error: true
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \
- --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')
- if [ -z "$run_id" ]; then
- echo "::warning::No successful timing run to fall back to, sharding without durations"
- exit 0
- fi
- gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \
- || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"
-
- - name: Restore per-segment test durations from cache
- # Also restored here so the snapshot below covers the per-segment plan files.
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: |
- .test_durations.core
- .test_durations.temporal
- key: posthog-segment-durations-${{ github.run_id }}
- restore-keys: |
- posthog-segment-durations-
-
- - name: Restore the events_json prune manifest from cache
- # The timing workflow rebuilds it from the newest hourly recording. A miss
- # leaves the file absent, and the events_json rows then run every listed test.
- if: github.event_name == 'pull_request'
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: events-schema-prune.json
- key: posthog-events-schema-prune-${{ github.run_id }}
- restore-keys: |
- posthog-events-schema-prune-
-
- - name: Snapshot the sharding plan for this run
- # The timing caches float to their newest entry, so a re-run can shard
- # differently from attempt 1 and, under "re-run failed jobs", a failing
- # test can move to a shard that never re-runs while the gate goes green.
- # Every test job applies this snapshot instead, so all attempts shard
- # identically. Best-effort (consumers fall back to the floating caches);
- # overwrite lets a "re-run all jobs" attempt re-snapshot.
- continue-on-error: true
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- with:
- name: test-durations-plan
- path: |
- .test_durations
- .test_durations.core
- .test_durations.temporal
- events-schema-prune.json
- include-hidden-files: true
- overwrite: true
- # The per-segment files and the prune manifest are cache-only and may legitimately be absent.
- if-no-files-found: warn
- # GitHub allows re-runs for 30 days.
- retention-days: 30
-
- # The backend test selector picks the Django tests this PR needs, and on a legacy
- # diff its product list narrows the product matrix too: turbo-discover.js cannot
- # see which products a posthog/ or ee/ edit reaches, and the selector can, because
- # it follows the import graph into products/. Both matrices read one verdict, so
- # they can never disagree about whether the selection is trustworthy.
- # Only on a legacy diff: turbo-discover.js trusts the selection only when the paths
- # filter saw a legacy edit, and falls back without a selector run otherwise. The
- # DISABLE_BACKEND_TEST_SELECTION kill switch skips the run too; the script still
- # reports the `disabled` reason from its own env.
- - name: Install uv
- # Optional setup: a failure here must fall back to the full matrices like a
- # selector failure does, not fail the job that gates every test job.
- continue-on-error: true
- if: env.SELECTION_APPLIES == 'true' && needs.changes.outputs.legacy == 'true' && vars.DISABLE_BACKEND_TEST_SELECTION != 'true'
- uses: ./.github/actions/setup-uv
- with:
- enable-cache: false
-
- - name: Run the backend test selector
- id: select
- # A selector failure or hang must not fail this job, which gates every test job
- # in the workflow — turbo-discover.js falls back to the full matrices instead.
- # The step timeout bounds a hang well inside the job's, so the fallback still
- # reaches the matrices rather than taking the whole job down with it.
- continue-on-error: true
- timeout-minutes: 8
- if: env.SELECTION_APPLIES == 'true' && needs.changes.outputs.legacy == 'true' && vars.DISABLE_BACKEND_TEST_SELECTION != 'true'
- shell: bash
- run: |
- set -euo pipefail
- uv run tools/snob_backend_test_selection_shadow.py \
- --base-ref "$TURBO_SCM_BASE" \
- > /tmp/selection.json
-
- - name: Discover products to test
- id: discover
- env:
- # On pushes to master, always run everything.
- # On PRs, use the path filter to detect legacy changes.
- LEGACY_CHANGED: ${{ github.event_name != 'pull_request' || needs.changes.outputs.legacy }}
- # Empty when the selector did not run or failed, which falls back to the
- # full matrices on a ready PR and skips them on a draft.
- SELECTION_JSON: ${{ steps.select.outcome == 'success' && '/tmp/selection.json' || '' }}
- # Kill switch — set the DISABLE_BACKEND_TEST_SELECTION repo variable to put
- # every PR back on the full matrices without a code change.
- SELECTION_DISABLED: ${{ vars.DISABLE_BACKEND_TEST_SELECTION || 'false' }}
- # A draft has a later ready run to defer to, so an untrusted selection skips
- # the heavy matrices there instead of running all of them.
- PR_DRAFT: ${{ github.event.pull_request.draft }}
- RUN_NEW_EVENTS_SCHEMA: ${{ contains(github.event.pull_request.labels.*.name, 'test-new-events-schema') || github.event_name == 'workflow_dispatch' }}
- SCHEMA_CHANGED: ${{ github.event_name != 'pull_request' || needs.changes.outputs.schema }}
- TURBO_SCM_HEAD: ${{ github.sha }}
- # Kill switch — drop comma-separated products from the matrix; empty = run all.
- SKIP_PRODUCT_TESTS: ${{ vars.SKIP_PRODUCT_TESTS || '' }}
- run: |
- # turbo-discover.js uses Turbo's Git affectedness to detect
- # changed products. Non-isolated product changes trigger the
- # full suite (all products + Django).
- RESULT=$(node .github/scripts/turbo-discover.js)
- echo "Result: $RESULT"
- echo "matrix=$(echo "$RESULT" | jq -c '.matrix')" >> $GITHUB_OUTPUT
- echo "run_legacy=$(echo "$RESULT" | jq -r '.run_legacy')" >> $GITHUB_OUTPUT
- echo "run_legacy_reason=$(echo "$RESULT" | jq -r '.run_legacy_reason // empty')" >> "$GITHUB_OUTPUT"
- echo "django_shards=$(echo "$RESULT" | jq -c '.django_shards // empty')" >> $GITHUB_OUTPUT
- echo "$RESULT" | jq -r '"mode=\(.selection.mode)", "selection=\(.selection | tojson)", "telemetry=\(.telemetry | tojson)"' >> "$GITHUB_OUTPUT"
-
- - name: Upload selection artifact
- # test-selection-verdict reads it instead of running the selector again. An
- # upload failure must not fail this job; the verdict regenerates the selection.
- continue-on-error: true
- if: always() && steps.select.outcome == 'success'
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- with:
- name: select-tests-output
- path: /tmp/selection.json
- if-no-files-found: ignore
- retention-days: 14
-
- build-product-test-matrix:
- name: Build product test matrix
- needs: [turbo-discover]
- if: >-
- !cancelled() &&
- needs.turbo-discover.result == 'success' &&
- needs.turbo-discover.outputs.matrix != '[]' &&
- needs.turbo-discover.outputs.matrix != ''
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 5
- outputs:
- include: ${{ steps.build.outputs.include }}
- steps:
- - name: Build product matrix include list
- id: build
- env:
- PRODUCT_MATRIX_JSON: ${{ needs.turbo-discover.outputs.matrix }}
- # The new-events-schema variants double the matrix, so they run only on
- # opted-in PRs and manual dispatches. Other runs cover the product paths
- # in .github/new-events-schema-targets.txt with the Django matrix's
- # events_json rows.
- RUN_NEW_EVENTS_SCHEMA: ${{ contains(github.event.pull_request.labels.*.name, 'test-new-events-schema') || github.event_name == 'workflow_dispatch' }}
- run: |
- if [[ "$RUN_NEW_EVENTS_SCHEMA" == "true" ]]; then
- schema_modes='[false, true]'
- else
- schema_modes='[false]'
- echo "Skipping new-events-schema variants (RUN_NEW_EVENTS_SCHEMA is off)"
- fi
- include=$(jq -cn --argjson matrix "$PRODUCT_MATRIX_JSON" --argjson schema_modes "$schema_modes" '
- [
- $matrix[] as $entry
- | $schema_modes[] as $new_events_schema
- | $entry + {"new-events-schema": $new_events_schema}
- ]
- ')
- echo "include=$include" >> "$GITHUB_OUTPUT"
- echo "Product test matrix size: $(jq -r 'length' <<< "$include")"
-
- # Runs product tests in parallel — one matrix job per group
- # Each job gets its own runner + Docker stack, so no shared DB conflicts
- # Small products (< 50 tests) are grouped into a single job to avoid setup overhead
- # A status function is required to inspect dependency results after failures.
- # Use !cancelled(), not always(), so superseded matrix jobs still receive cancellation.
- turbo-tests:
- needs: [changes, turbo-discover, detect-snapshot-mode, build-product-test-matrix, get_clickhouse_versions]
- if: >-
- !cancelled() &&
- (github.event.pull_request.draft != true ||
- startsWith(github.head_ref, 'trunk-merge/') ||
- contains(github.event.pull_request.labels.*.name, 'run-ci-backend')) &&
- needs.turbo-discover.result == 'success' &&
- needs.build-product-test-matrix.result == 'success' &&
- needs.build-product-test-matrix.outputs.include != '[]' &&
- needs.build-product-test-matrix.outputs.include != ''
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 40
- name: Product tests (${{ matrix.group }}, events schema ${{ matrix.new-events-schema && 'json' || 'legacy' }})
- strategy:
- fail-fast: false
- matrix:
- include: ${{ fromJson(needs.build-product-test-matrix.outputs.include) }}
- env:
- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
- # Start Docker early (before dependency installs) so containers can pull
- # images and initialize while we install deps. This matches the pattern
- # used by core-tests/django and makes the later wait step negligible.
- - name: Clean up data directories
- run: |
- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
- continue-on-error: true
-
- - name: Log in to Docker Hub
- continue-on-error: true
- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
- with:
- username: ${{ vars.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
- - name: Start services
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- COMPOSE_PROFILES: temporal,azure
- CLICKHOUSE_SERVER_IMAGE: ${{ needs.get_clickhouse_versions.outputs.local_image }}
- run: |
- cp posthog/user_scripts/latest_user_defined_function.xml docker/clickhouse/user_defined_function.xml
- bin/ci-wait-for-docker launch --background --down \
- db redis7 clickhouse zookeeper kafka objectstorage \
- temporal elasticsearch objectstorage-azure
-
- # Dedicated app: this product matrix alone is ~half the offloadable
- # setup-action volume, enough to threaten any shared 15k/hr bucket.
- - name: Mint setup-action GitHub token
- id: setup-gh-token
- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ vars.GH_APP_POSTHOG_PRODUCT_TESTS_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_PRODUCT_TESTS_PRIVATE_KEY }}
- skip-token-revoke: true
-
- # Sequential, not a `parallel:` block. Each of these writes $GITHUB_PATH,
- # and concurrent writes crash the GitHub Actions runner ("Collection was
- # modified" or a missing-key error). The runner fails the step even though
- # the tool installed fine, so the job dies in setup with nothing to debug.
- #
- # turbo launches each backend:test script through pnpm, so pnpm and
- # Node must stay on PATH even though nothing here reads the workspace.
- - name: Set up pnpm and Node
- uses: ./.github/actions/setup-pnpm
- with:
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- - name: Set up Python
- uses: ./.github/actions/setup-python-cached
- with:
- python-version: 3.14.7
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- # Needs pnpm from the step above.
- - name: Install turbo
- uses: ./.github/actions/setup-turbo
-
- - name: Install uv
- id: setup-uv
- uses: ./.github/actions/setup-uv
-
- - name: Install SAML dependencies
- if: steps.setup-uv.outputs.cache-hit != 'true'
- run: sudo rm -f /etc/apt/sources.list.d/*twingate* && sudo apt-get update && sudo apt-get install -y libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-
- - name: Install sqlx-cli
- uses: ./.github/actions/setup-sqlx-cli
-
- - name: Install Python dependencies
- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
-
- - name: Add service hostnames to /etc/hosts
- run: echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage temporal" | sudo tee -a /etc/hosts
-
- - name: Set up needed files
- run: |
- mkdir -p frontend/dist
- touch frontend/dist/index.html frontend/dist/layout.html frontend/dist/exporter.html
- ./bin/download-mmdb
-
- - name: Wait for Docker services
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- COMPOSE_PROFILES: temporal,azure
- CLICKHOUSE_SERVER_IMAGE: ${{ needs.get_clickhouse_versions.outputs.local_image }}
- # objectstorage answers InvalidAccessKeyId until its bootstrap registers identities.
- run: bin/ci-wait-for-docker wait objectstorage
-
- - name: Restore schema cache from master
- if: ${{ github.event_name == 'pull_request' && needs.turbo-discover.outputs.schema_cache_key != '' }}
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: schema.sql.gz
- # Prefer the shared content key; fall back to the exact merge-base SHA key,
- # then to the newest dump of any migration set. Right after a migration
- # lands on master both exact keys miss until ci-backend re-dumps, and a
- # from-scratch migrate costs ~17 minutes per shard; pytest's keepdb
- # migrate tops a slightly stale dump up in seconds instead.
- key: ${{ needs.turbo-discover.outputs.schema_migrations_key || needs.turbo-discover.outputs.schema_cache_key }}
- restore-keys: |
- ${{ needs.turbo-discover.outputs.schema_cache_key }}
- ${{ needs.turbo-discover.outputs.schema_prefix_key }}
-
- - name: Prime test_posthog from cached schema
- # Treat cache problems as misses; the migrate step below rebuilds the DB
- # whenever this one reports restored=false.
- id: prime-schema
- if: ${{ github.event_name == 'pull_request' }}
- run: |
- echo "restored=false" >> "$GITHUB_OUTPUT"
- if [ ! -f schema.sql.gz ]; then
- echo "::notice::Schema cache miss; the migrate step will build test_posthog"
- exit 0
- fi
- mkdir -p .postgres-backups
- mv schema.sql.gz .postgres-backups/schema-latest.sql.gz
- if ./bin/hogli db:restore-test-db; then
- echo "restored=true" >> "$GITHUB_OUTPUT"
- else
- echo "::warning::Schema restore failed (stale/incompatible cached dump?); the migrate step will build test_posthog"
- fi
-
- - name: Migrate test_posthog from scratch
- # Any run without a restored schema cache (master, dispatches, PR cache
- # misses) replays the full migration chain here, in a process that exits
- # before tests: the executor's memory high-water stays resident in
- # whichever process runs it, and pytest must not carry it through the
- # test phase. pytest --reuse-db adopts the DB like the PR prime above;
- # any failure falls back to the in-pytest migrate.
- if: ${{ github.event_name != 'pull_request' || steps.prime-schema.outputs.restored != 'true' }}
- env:
- DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog
- run: |
- admin_psql() { docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres "$@"; }
- if ! admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" -c "CREATE DATABASE test_posthog;"; then
- echo "::warning::Could not recreate test_posthog; pytest --reuse-db will run the full migrate itself"
- exit 0
- fi
- if ! python manage.py migrate; then
- echo "::warning::Out-of-process migrate failed; pytest --reuse-db will run the full migrate itself"
- admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" || true
- fi
-
- - name: Register Temporal search attributes
- # Some product groups (data_warehouse, batch_exports) connect to a live Temporal
- # via sync_connect() in their tests, so keep a readiness gate before tests run.
- # By this point Temporal has warmed up in the background, so the wait is negligible.
- # data_warehouse tests use custom search attributes (PostHogTeamId, PostHogOrgId,
- # PostHogDagId) when scheduling materialization, so they must be registered
- # before tests run.
- # If the command hangs, SIGABRT makes faulthandler print the Python stack of each
- # thread before the process exits, so the log shows where it stopped.
- timeout-minutes: 10
- run: |
- bin/wait-for-docker temporal
- PYTHONFAULTHANDLER=1 timeout --signal=ABRT --kill-after=30s 3m python manage.py register_temporal_search_attributes
-
- - name: Restore test durations from cache
- # pytest-split distributes this product's tests across shards using
- # .test_durations. Restore the newest cache; the fallback step below covers
- # a miss.
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: .test_durations
- # run_id never matches — forces the prefix restore-key to the newest entry.
- key: posthog-test-durations-${{ github.run_id }}
- restore-keys: |
- posthog-test-durations-
-
- - name: Fall back to the last published durations
- # .test_durations is not in the repo. The timing workflow refreshes the
- # cache several times a day and publishes the same files as an artifact
- # of its own runs. On a cache miss (retention lapsed, timing workflow
- # down for two weeks) take the newest artifact; if that is gone too,
- # pytest-split splits by count and turbo-discover sizes from file counts
- # until the next timing run.
- if: ${{ hashFiles('.test_durations') == '' }}
- continue-on-error: true
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \
- --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')
- if [ -z "$run_id" ]; then
- echo "::warning::No successful timing run to fall back to, sharding without durations"
- exit 0
- fi
- gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \
- || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"
-
- - name: Download the run's sharding plan snapshot
- # Same pinning as the Django job; falls back to the floating cache.
- id: pinned-plan
- continue-on-error: true
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- name: test-durations-plan
- path: /tmp/test-durations-plan
-
- - name: Apply the pinned sharding plan
- if: steps.pinned-plan.outcome == 'success'
- shell: bash
- run: |
- if [ ! -f /tmp/test-durations-plan/.test_durations ]; then
- echo "::warning::Sharding-plan snapshot is incomplete, keeping the floating cache restore"
- exit 0
- fi
- cp /tmp/test-durations-plan/.test_durations .test_durations
- echo "Sharding plan pinned to this run's snapshot"
-
- - name: Warn when a re-run reshards unpinned
- if: github.run_attempt != '1' && steps.pinned-plan.outcome != 'success'
- shell: bash
- run: echo "::warning::Sharding-plan snapshot unavailable, this re-run reshards from the floating timing cache and may not re-execute tests that failed in earlier attempts"
-
- - name: Run product tests
- id: run-product-tests
- # continue-on-error so the quarantine gate below is the verdict — it passes when
- # every failure is an already-quarantined flake, fails otherwise.
- continue-on-error: true
- # --force: discover already decided this product needs testing, skip turbo cache
- # --log-order=stream: stream pytest output live instead of buffering until completion
- # legs: one turbo invocation per entry, each with its own pytest-split flags
- # (e.g. "-- --splits 3 --group 1"). A job holds several legs when a split
- # product's last shard had budget left over for whole small products, and
- # pytest-split flags must not leak from that shard onto its job-mates.
- env:
- PRODUCT_LEGS: ${{ toJSON(matrix.legs) }}
- LEGACY_FILTERS: ${{ matrix.filters }}
- LEGACY_PYTEST_ARGS: ${{ matrix.pytest_args }}
- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- # Person-on-events v2 reads are the production default for every project
- # created since 2024-06-14, so product tests run with the mode on, as the
- # Core Django rows do. Without it a product query runner is only ever
- # tested in the legacy joined mode, and its snapshots record SQL that no
- # recent project executes. Tests that need the legacy mode pin it
- # themselves with override_settings.
- # marker: product-poe-default. The run block keeps legacy mode if the
- # checked-out workflow lacks this marker and its matching snapshots.
- PERSON_ON_EVENTS_V2_ENABLED: 'true'
- # products/tasks/backend/temporal moved here from the Django Temporal
- # segment, and its conftest and workflows talk to Modal. Injected only
- # when that directory changed, matching the gate the Django job used.
- MODAL_TOKEN_ID: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_ID || '' }}
- MODAL_TOKEN_SECRET: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_SECRET || '' }}
- AWS_S3_ALLOW_UNSAFE_RENAME: 'true'
- # --reuse-db: keep the test database between sequential product runs to avoid
- # ClickHouse drop/create race conditions with ReplicatedMergeTree ZK metadata.
- # --durations-path: turbo runs products from products/<name>/, so pytest-split
- # needs an explicit path back to the repo-root durations file for balanced sharding.
- # --junitxml: written relative to each product's dir (turbo's cwd), so every
- # product in the group gets its own products/<name>/junit-product.xml.
- # On master, also collect timing data for pytest-split sharding.
- PYTEST_ADDOPTS: >-
- --reuse-db
- --durations-path ../../.test_durations
- --junitxml=junit-product.xml
- ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && '--snapshot-update --snapshot-warn-unused' || '' }}
- ${{ github.ref == 'refs/heads/master' && '--store-durations' || '' }}
- ${{ github.event_name == 'pull_request' && '--cov=backend --cov-report=' || '' }}
- # turbo runs each product's backend:test with the product dir as CWD, so --cov=backend
- # scopes coverage to that product and saves its data to products/<name>/.coverage. This works
- # for bin-packed buckets (each product writes its own file) and split products, whose data
- # the report job combines.
- # sysmon: Python 3.12+'s low-overhead coverage backend, ~a few % vs ~20% for the C tracer.
- COVERAGE_CORE: sysmon
- shell: bash
- run: |
- set +e
- # Enable retries only when the checkout includes retry diagnostics.
- if grep -qF '"posthog.runner_name"' posthog/conftest.py; then
- export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --force-reruns 1 --reruns-delay 1 -r fEsxXR"
- fi
- overall=0
- # A checkout without the matching snapshots must use legacy mode.
- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: product-poe-default" .github/workflows/ci-backend.yml; then
- echo "::notice::Checked-out revision predates the person-on-events default, running product tests in legacy mode"
- export PERSON_ON_EVENTS_V2_ENABLED=false
- fi
- # Accept a selector that returns the older flat matrix shape.
- if [ -z "$PRODUCT_LEGS" ] || [ "$PRODUCT_LEGS" = "null" ]; then
- PRODUCT_LEGS=$(jq -cn --arg f "$LEGACY_FILTERS" --arg a "$LEGACY_PYTEST_ARGS" \
- '[{filters: $f, pytest_args: $a}]')
- echo "Matrix entry carries no legs; running the legacy single invocation"
- fi
- leg_count=$(jq 'length' <<< "$PRODUCT_LEGS")
- # Neither shape present means the job would pass having run no tests, and
- # the rollup gate only reads this step's outcome. Fail loudly instead.
- if [ -z "$leg_count" ] || [ "$leg_count" -lt 1 ]; then
- echo "::error::No test legs in this matrix entry; refusing to pass without running tests"
- exit 1
- fi
- for ((leg_index = 0; leg_index < leg_count; leg_index++)); do
- leg_filters=$(jq -r ".[$leg_index].filters" <<< "$PRODUCT_LEGS")
- leg_args=$(jq -r ".[$leg_index].pytest_args" <<< "$PRODUCT_LEGS")
- echo "::group::turbo backend:test $leg_filters $leg_args"
- # Word splitting is intended here: both variables carry several flags.
- # shellcheck disable=SC2086
- turbo run backend:test $leg_filters --concurrency=1 --output-logs=full --force --log-order=stream $leg_args
- exit_code=$?
- echo "::endgroup::"
- if [ $exit_code -eq 5 ]; then
- echo "No tests collected for this leg, this is expected when splitting tests"
- elif [ $exit_code -ne 0 ]; then
- overall=$exit_code
- fi
- done
- set -e
- exit $overall
-
- # Stages every product's report under a per-product filename. Pass a report path to
- # exclude a product from Trunk quarantine; this step then owns that product's verdict.
- - name: Prepare product JUnit for Trunk
- id: prepare_product_junit
- if: always()
- shell: bash
- env:
- PRODUCT_FILTERS: ${{ matrix.filters }}
- run: |
- bash .github/scripts/prepare-product-junit-for-trunk.sh \
- trunk-junit \
- "$PRODUCT_FILTERS"
-
- # Best-effort Trunk upload (continue-on-error); the "Fail on test failure" step below is
- # the verdict, so a Trunk outage can't red a passing shard. Internal PRs only (needs the
- # secret).
- # TRUNK_UPLOAD_ENABLED is the master kill-switch: when it is not 'true' this gate
- # skips, so nothing uploads and no known flakes are quarantined. TRUNK_QUARANTINE_ENABLED
- # splits the two — it defaults off, so set it to 'true' to let the verdict step below mask
- # known flakes; otherwise any real failure reds the job (quarantining needs upload on too).
- - name: Quarantine gate
- id: quarantine_gate
- continue-on-error: true
- if: ${{ !cancelled() && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}
- uses: ./.github/actions/trunk-quarantine-gate
- with:
- junit-paths: trunk-junit/junit-product-*.xml
- test-collection-id: C0LrorjW # backend-product
- # A native-JSON leg uploads under its own variant, so a failure in that mode
- # does not read as a flake of the legacy test with the same id.
- variant: ${{ matrix.new-events-schema && 'events-json' || '' }}
- previous-step-outcome: ${{ steps.run-product-tests.outcome }}
- token: ${{ secrets.TRUNK_API_TOKEN }}
-
- # Verdict: red a real failure the gate didn't clear as a quarantined flake. != 'success'
- # also covers the skipped gate on fork/Dependabot. Sits before the timing-data upload so
- # a real failure still skips it, as it did before the test step got continue-on-error.
- - name: Fail on test failure
- if: ${{ !cancelled() && (steps.prepare_product_junit.outcome == 'failure' || (steps.run-product-tests.outcome == 'failure' && (vars.TRUNK_QUARANTINE_ENABLED != 'true' || steps.quarantine_gate.outcome != 'success'))) }}
- shell: bash
- run: exit 1
-
- # Feeds the django_tests gate's per-test failure rollup. Rename each file to carry
- # its product so the name survives upload-artifact's least-common-ancestor path
- # collapse (same trick as "Stage product coverage" below).
- - name: Stage product JUnit
- if: always()
- shell: bash
- run: |
- mkdir -p junit-staging
- for f in products/*/junit-product.xml; do
- [ -f "$f" ] || continue
- cp "$f" "junit-staging/junit-product-$(basename "$(dirname "$f")").xml"
- done
- # Separate artifact prefix from junit-results-backend-* so consumers can select
- # product JUnit by download pattern; the product name rides in each inner filename
- # (junit-product-<name>.xml). The failure rollup and timing reporter consume both.
- - name: Upload product test results
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- if: always()
- with:
- name: product-junit-results-${{ strategy.job-index }}
- path: junit-staging/junit-product-*.xml
- if-no-files-found: ignore
- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwise
-
- - name: Upload timing data
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- # Only the legacy-schema variant uploads durations — both schema variants run the
- # same test IDs, and two artifacts for the same tests would race in the merge.
- # A job whose products have no backend tests writes no JUnit, so it has no fresh
- # timings either. Its artifact would hold only the restored map, and the timing
- # refresh then rejects the whole Products slice because the timing and JUnit shard
- # sets differ.
- if: ${{ github.ref == 'refs/heads/master' && !matrix.new-events-schema && hashFiles('junit-staging/junit-product-*.xml') != '' }}
- with:
- name: timing_data-Products-${{ strategy.job-index }}
- path: .test_durations
- include-hidden-files: true
- retention-days: 2
-
- - name: Stage product coverage
- # Rename each products/<name>/.coverage to <name>.coverage so the product name
- # survives upload-artifact's least-common-ancestor path collapse.
- if: ${{ always() && github.event_name == 'pull_request' }}
- shell: bash
- run: |
- mkdir -p cov-staging
- for f in products/*/.coverage; do
- [ -f "$f" ] || continue
- product=$(basename "$(dirname "$f")")
- cp "$f" "cov-staging/${product}.coverage"
- done
-
- - name: Upload product coverage
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- # always(): a failing test shard still has partial coverage worth reporting.
- if: ${{ always() && github.event_name == 'pull_request' }}
- with:
- name: coverage-products-${{ strategy.job-index }}
- path: cov-staging/*.coverage
- if-no-files-found: ignore
- retention-days: 2
- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwise
-
- - name: Verify new snapshots for flakiness
- if: ${{ always() && needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}
- shell: bash
- # The reruns must write the same schema mode's snapshot file as the main
- # run, or the new-schema file removed before each rerun never comes back.
- env:
- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- run: |
- .github/scripts/verify-new-snapshots.sh
-
- - name: Generate snapshot patch
- if: ${{ always() && needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}
- shell: bash
- env:
- NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- run: |
- mkdir -p /tmp/patches
- git add -N '*.ambr' || true
- # Same leg scoping as the django job: a leg may only ship its own
- # schema mode's snapshot files (see the comment there).
- if [[ "$NEW_EVENTS_SCHEMA" == "true" ]]; then
- ambr_pathspec=('*.new_events_schema.ambr')
- else
- ambr_pathspec=('*.ambr' ':(exclude)*.new_events_schema.ambr')
- fi
- if ! git diff --quiet -- "${ambr_pathspec[@]}" 2>/dev/null; then
- git diff --binary --full-index -- "${ambr_pathspec[@]}" > /tmp/patches/backend-Products-${{ strategy.job-index }}.patch
- echo "Generated patch with $(wc -l < /tmp/patches/backend-Products-${{ strategy.job-index }}.patch) lines"
- else
- echo "No snapshot changes to patch"
- fi
-
- - name: Upload snapshot patch
- if: ${{ always() && needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- with:
- name: snapshot-patch-Products-${{ strategy.job-index }}
- path: /tmp/patches/
- if-no-files-found: ignore
- retention-days: 1
-
- # Lightweight repo-wide checks that only need Python + uv (no Docker/DB).
- # Consolidates checks that previously each spun up their own runner.
- # Every PostHog SDK this repo installs must stay on its pinned major; the release
- # bots rewrite manifests across majors, so this is the block. Stdlib only, no venv.
- sdk-major-guard:
- needs: [changes]
- if: needs.changes.outputs.sdk_manifests == 'true'
- timeout-minutes: 5
- permissions: {}
- name: SDK major-version guard
- runs-on: ubuntu-24.04
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- - name: Check every SDK stays on its pinned major
- run: python3 .github/scripts/check_sdk_majors.py
-
- repo-checks:
- needs: [changes]
- # product_yamls: ownership-only PRs still need the owners.yaml lint below.
- if: (needs.changes.outputs.backend == 'true' || needs.changes.outputs.product_yamls == 'true' || needs.changes.outputs.repo_invariants == 'true')
- timeout-minutes: 10
- outputs:
- deterministic_failure: ${{ steps.deterministic-failure.outputs.deterministic_failure || 'false' }}
- permissions:
- contents: read
- pull-requests: read # list PR files for the deleted-migration check
- name: Repo checks (depot-ubuntu-24.04)
- runs-on: depot-ubuntu-24.04
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
- - name: Mint setup-action GitHub token
- id: setup-gh-token
- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}
- skip-token-revoke: true
-
- - name: Set up Python
- uses: ./.github/actions/setup-python-cached
- with:
- python-version: 3.14.7
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- - name: Install uv
- uses: ./.github/actions/setup-uv
- with:
- enable-cache: false
-
- - name: Install Python dependencies
- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
-
- # Above the boundary: a gh-api flake here is retryable setup, not a
- # run-cancelling deterministic failure. The classifier runs below.
- - name: Fetch deleted migration paths
- id: fetch-deleted-migrations
- if: github.event_name == 'pull_request' && needs.changes.outputs.migrations_any == 'true'
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- run: .github/scripts/list-removed-renamed-paths.sh > "$RUNNER_TEMP/deleted-migration-paths.txt"
-
- # Above the boundary for the same reason. product:lint compares the crossings ledger
- # with the merge commit's first parent, so the clone needs one more level of history.
- - name: Fetch the pull request base
- if: github.event_name == 'pull_request'
- run: |
- git remote get-url origin >/dev/null 2>&1 || git remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
- git fetch --no-tags --depth=2 origin "$GITHUB_SHA"
-
- # Boundary between network-dependent setup and the deterministic
- # checks below. The "Flag deterministic failure" step keys off this
- # step's outcome so a transient setup failure doesn't cancel the run.
- - name: Mark setup complete
- id: setup-complete
- run: echo "Setup complete — any failure below is a deterministic check failure"
-
- # Below the boundary so a real deletion trips the early-cancel gate (see
- # safe-django-migrations.md). Gated on the fetch outcome so a skipped fetch
- # skips this too, rather than reading a file it never wrote.
- - name: Check deleted migrations
- if: steps.fetch-deleted-migrations.outcome == 'success'
- run: ./bin/hogli lint:migration-deletions < "$RUNNER_TEMP/deleted-migration-paths.txt"
-
- - name: Bootstrap scaffold product
- run: ./bin/hogli product:bootstrap spline_reticulator --non-interactive
-
- - name: Lint product structure
- env:
- CROSSINGS_LEDGER_BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || '' }}
- run: ./bin/hogli product:lint --all
-
- # Distributed owners.yaml tree: schema, product.yaml/owners.yaml
- # conflicts, dead rule globs, coverage (warn-only). No --live, so no
- # secrets — fork PRs get validated too (the --live slug check runs in
- # the fork-gated validate-product-yamls job instead).
- - parallel:
- - name: Lint owners.yaml files
- run: ./bin/hogli owners:lint
-
- - name: Check version specifiers
- run: python .github/scripts/check-version-specifiers.py
-
- - name: Check IDOR model coverage
- run: python .github/scripts/check-idor-model-coverage.py
-
- - name: Check DWH presentation layer stays source-agnostic
- run: python .github/scripts/check-dwh-source-agnostic.py
-
- - name: Check operator parity
- run: python .github/scripts/check-operator-parity.py
-
- - name: Check module boundaries (tach)
- # Two passes: tests are in the interface graph but not the dependency graph.
- # A branch that predates the command still excludes tests in tach.toml, so
- # the old single pass is the right check there.
- run: |
- if ./bin/hogli lint:tach --help >/dev/null 2>&1; then
- ./bin/hogli lint:tach
- else
- tach check --dependencies --interfaces
- fi
-
- - name: Check product facade enforcement (import-linter)
- run: lint-imports
-
- # These guards take the whole repo as input (import graph after
- # django.setup(), signal registries, repo-wide AST scans), so diff-based
- # test selection can't pick them and the Django matrix skips them on
- # products-only diffs and drafts. They need no service containers.
- # Branches that predate the directory still carry the guards at their
- # old paths, so fall back to those rather than failing on a missing arg.
- - name: Run repo invariants (whole-repo pytest guards)
- run: |
- if [ -d posthog/test/repo_invariants ]; then
- pytest -p no:cacheprovider posthog/test/repo_invariants
- else
- pytest -p no:cacheprovider posthog/test/test_startup_import_budget.py posthog/test/test_dataclass_defaults.py
- fi
-
- # Deterministic = a static check above failed after setup completed.
- # A retry can't fix it, so this signals the cancellation gate to stop the run.
- - name: Flag deterministic failure
- id: deterministic-failure
- if: failure() && steps.setup-complete.outcome == 'success'
- run: echo "deterministic_failure=true" >> "$GITHUB_OUTPUT"
-
- cancel-backend-on-repo-check-failure:
- needs: [repo-checks]
- if: >
- always() &&
- needs.repo-checks.outputs.deterministic_failure == 'true' &&
- github.event_name == 'pull_request' &&
- github.event.pull_request.head.repo.full_name == github.repository
- timeout-minutes: 5
- permissions:
- contents: read
- actions: write
- name: Cancel Backend CI after repo check failure
- runs-on: ubuntu-24.04
- steps:
- - name: Cancel run
- env:
- GH_TOKEN: ${{ github.token }}
- GH_REPO: ${{ github.repository }}
- run: |
- echo "::notice title=Repo checks failed deterministically::Canceling remaining Backend CI jobs. A retry cannot fix this failure; push a fix to start a new run."
- gh run cancel ${{ github.run_id }}
-
- # Validates product.yaml owners against PostHog/posthog collaborator teams.
- # Gated on the product_yamls paths filter so we only spend a GitHub API call
- # when ownership actually changes. The scoped form (positional product names)
- # means a PR can't trip on pre-existing stale ownership in unrelated yamls.
- # Skipped for fork PRs — app-token secrets aren't available there, and forks
- # can't introduce new product.yaml entries that ship to master without a
- # same-repo PR going through this check anyway.
- validate-product-yamls:
- needs: [changes]
- if: >
- needs.changes.outputs.product_yamls == 'true' &&
- github.event_name == 'pull_request' &&
- github.event.pull_request.head.repo.full_name == github.repository
- timeout-minutes: 5
- name: Validate product.yaml owners (ubuntu-24.04)
- runs-on: ubuntu-24.04
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
- # Default GITHUB_TOKEN can't list /repos/{repo}/teams (needs org-level
- # `members: read`). Reuse the assign-reviewers app — same secret, same
- # use case. `owner: PostHog` scopes the issued token at the org so
- # org-level permissions actually flow through; otherwise the token is
- # repo-scoped and the org scope gets dropped.
- - name: Get app token
- id: app-token
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ secrets.GH_APP_POSTHOG_ASSIGN_REVIEWERS_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_ASSIGN_REVIEWERS_PRIVATE_KEY }}
- owner: ${{ github.repository_owner }}
- repositories: ${{ github.event.repository.name }}
-
- # Plain setup-python: this PR-only job runs on ubuntu-24.04, whose cache
- # store no master job ever saves the python tool cache to, so the
- # setup-python-cached restore would miss on every run.
- - name: Set up Python
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: 3.14.7
- # Reuse the app token already minted above instead of a second mint.
- token: ${{ steps.app-token.outputs.token || github.token }}
-
- - name: Install uv
- uses: ./.github/actions/setup-uv
- with:
- enable-cache: false
-
- - name: Install Python dependencies
- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
-
- - name: Validate changed product.yaml owners
- env:
- GH_TOKEN: ${{ steps.app-token.outputs.token }}
- CHANGED_FILES: ${{ needs.changes.outputs.product_yamls_files }}
- run: |
- # CHANGED_FILES is space-separated (paths-filter list-files: escape).
- # Each entry is products/<name>/product.yaml — strip to <name>.
- names=$(echo "$CHANGED_FILES" | tr ' ' '\n' | sed -n 's|^products/\([^/]*\)/product.yaml$|\1|p' | xargs)
- # Drop products whose product.yaml was deleted in this PR (e.g. the
- # product moved out of products/) — there are no owners left to validate.
- names=$(for n in $names; do [ -f "products/$n/product.yaml" ] && printf '%s ' "$n"; done | xargs)
- if [ -z "$names" ]; then
- echo "No existing product.yaml files in diff after filtering"
- else
- ./bin/hogli product:lint:owners $names
- fi
-
- - name: Validate owners.yaml identities against GitHub
- env:
- GH_TOKEN: ${{ steps.app-token.outputs.token }}
- CHANGED_FILES: ${{ needs.changes.outputs.product_yamls_files }}
- # Scoped to the diff: unscoped, one stale slug fails every owners.yaml PR.
- # xargs re-parses the paths-filter escape format, keeping escaped paths whole.
- run: printf '%s' "$CHANGED_FILES" | xargs -r ./bin/hogli owners:lint --live
-
- # Migration validation.
- # This job needs Docker + DB — it checks out master first to run baseline
- # migrations, then checks out the PR branch. OpenAPI type generation belongs
- # in check-openapi-types below.
- check-migrations:
- needs: [changes]
- # migration_replay is listed because it reaches files the other three miss:
- # products/db_routing.yaml is neither Python nor under a backend/ directory, and
- # a migration in a future top-level app matches none of them. Without it the
- # replay gate could be on while the job that runs it never starts.
- if: (needs.changes.outputs.backend == 'true' || needs.changes.outputs.migrations == 'true' || needs.changes.outputs.persons_sql == 'true' || needs.changes.outputs.migration_replay == 'true')
- # On master this job replays the full migration history from scratch to
- # produce the schema dump every PR job restores, and a merge queue batch that
- # changes a Django migration replays it a second time onto an empty database.
- # That replay grows with every merged migration, so the limit needs headroom
- # over two of them: it is there to cancel a hang, not to cap how long the
- # history takes.
- timeout-minutes: 60
- # The "Publish Migration risk check" step posts the check via the
- # posthog-tests app token, which carries its own checks:write
- # installation permission — so the check lands in that app's own
- # (otherwise-empty) check suite rather than floating under a random
- # run in the crowded github-actions suite. The job's GITHUB_TOKEN
- # therefore no longer needs checks:write. pull-requests:write is still
- # required for the migration sections in the CI report.
- permissions:
- contents: read
- pull-requests: write
-
- name: Validate migrations
- runs-on: depot-ubuntu-24.04
- env:
- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- clean: false
- - name: Clean up data directories with container permissions
- run: |
- # Use docker to clean up files created by containers
- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
- continue-on-error: true
-
- - name: Log in to Docker Hub
- continue-on-error: true
- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
- with:
- username: ${{ vars.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
- - name: Stop/Start stack with Docker Compose
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- run: |
- bin/ci-wait-for-docker launch --background --down db redis7 clickhouse
-
- - name: Mint setup-action GitHub token
- id: setup-gh-token
- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}
- skip-token-revoke: true
-
- - name: Set up Python
- uses: ./.github/actions/setup-python-cached
- with:
- python-version: 3.14.7
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- # Direct install: The job later checks out the PR head, which may predate the local action, so its post step would fail.
- - name: Install uv
- id: setup-uv
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- with:
- version: '0.13.4'
- enable-cache: true
- cache-dependency-glob: uv.lock
- save-cache: ${{ github.ref == 'refs/heads/master' }}
-
- - name: Install SAML (python3-saml) dependencies
- if: steps.setup-uv.outputs.cache-hit != 'true'
- run: |
- sudo rm -f /etc/apt/sources.list.d/*twingate*
- sudo apt-get update
- sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-
- - name: Install sqlx-cli
- uses: ./.github/actions/setup-sqlx-cli
-
- # First running migrations from master, to simulate the real-world scenario
- - name: Checkout master
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- ref: master
- clean: false
-
- - name: Install python dependencies for master
- run: |
- UV_PROJECT_ENVIRONMENT=.venv-master uv sync --frozen --dev --no-install-package pytest-split
-
- - name: Compute master schema cache key
- # Must match the migration-set computation in turbo-discover and the
- # push-side save below (inputs, hash, and key prefix). Keyed off the
- # master checkout (HEAD here) so a restored dump is exactly the state
- # that "Run migrations up to master" would build from scratch.
- if: github.event_name != 'push'
- id: master-schema-key
- run: |
- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' HEAD \
- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \
- | grep -vE '/(clickhouse|async_migrations)/migrations/' \
- | LC_ALL=C sort) || true
- # Routing decides which app labels are in the dump, so these inputs belong in the key.
- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)
- PG_IMAGE=$(git show "HEAD:docker-compose.base.yml" 2>/dev/null \
- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')
- if [ -z "$MIG_FILES" ]; then
- echo "key=" >> $GITHUB_OUTPUT
- else
- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)
- echo "key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT
- fi
-
- - name: Restore master schema dump from cache
- if: github.event_name != 'push' && steps.master-schema-key.outputs.key != ''
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: schema.sql.gz
- key: ${{ steps.master-schema-key.outputs.key }}
- # The exact key misses in the minutes between a master migration
- # landing and its dump being saved. The PR base commit's dump is
- # always older than the checkout, so migrate just tops it up.
- restore-keys: |
- posthog-schema-master-${{ github.event.pull_request.base.sha }}
-
- - name: Wait for Docker services
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- run: bin/ci-wait-for-docker wait --only db redis7 clickhouse
-
- - name: Prime posthog DB from cached schema
- # Turns "Run migrations up to master" below into a no-op instead
- # of a full migration run.
- if: github.event_name != 'push'
- run: |
- if [ ! -f schema.sql.gz ]; then
- echo "::notice::Master schema cache miss — the next step runs all migrations from scratch"
- exit 0
- fi
- # The load is atomic: on failure the transaction rolls back and
- # the fallback below recreates an empty DB for a full migrate.
- # ensure_migration_defaults re-seeds data-migration rows the
- # schema-only dump lacks.
- if gunzip -c schema.sql.gz | docker compose -f docker-compose.dev.yml exec -T db psql -q -v ON_ERROR_STOP=1 --single-transaction -U posthog posthog > /dev/null \
- && .venv-master/bin/python manage.py ensure_migration_defaults; then
- echo "Primed posthog DB from cached master schema"
- else
- echo "::warning::Cached schema restore failed — recreating empty DB; the next step runs all migrations from scratch"
- # Terminate lingering sessions first or DROP DATABASE fails
- # (same guard as _recreate_database in hogli's db_schema.py).
- docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'posthog' AND pid <> pg_backend_pid();"
- docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres -c "DROP DATABASE IF EXISTS posthog;"
- docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres -c "CREATE DATABASE posthog;"
- fi
- rm -f schema.sql.gz
-
- - name: Run migrations up to master
- run: |
- # Run Django migrations first (excluding managed=False models)
- .venv-master/bin/python manage.py migrate
- # Then run persons migrations using sqlx; comment out until we've merged
- # DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
- # sqlx database create
- # DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
- # sqlx migrate run --source rust/persons_migrations/
-
- # Now we can consider this PR's migrations
- - name: Checkout this PR
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- # For same-repo PRs, checkout the actual branch (not the merge commit)
- # so origin/master..HEAD reflects PR files. Fork PRs fall back
- # to the default merge commit.
- ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') && github.event.pull_request.head.ref || github.ref }}
- clean: false
-
- - name: Install python dependencies for this PR
- run: |
- UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
-
- - name: Check migrations and post SQL to CI report
- if: github.event_name == 'pull_request' && needs.changes.outputs.migrations == 'true'
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- CHANGED_FILES: ${{ needs.changes.outputs.migrations_review_files }}
- BASE_SHA: ${{ github.event.pull_request.base.sha }}
- run: |
- # The PR checkout above uses the PR branch, which on unrebased PRs
- # predates this script — skip the CI report section rather than fail.
- post_migration_section() {
- if [ -f .github/scripts/post-django-migration-section.mjs ]; then
- node .github/scripts/post-django-migration-section.mjs "$@"
- else
- echo "Skipping Django migration section — script not present on this checkout"
- fi
- }
-
- # If no migration files changed, exit
- if [ -z "$CHANGED_FILES" ]; then
- echo "No migration files changed"
- echo "No new Django migrations need SQL review." > /tmp/django-migration-sql.md
- post_migration_section sql clear ok "no new migrations" /tmp/django-migration-sql.md
- exit 0
- fi
-
- if [ -z "$BASE_SHA" ]; then
- echo "::warning::BASE_SHA is empty — all changed migrations will be treated as new"
- else
- # Ensure the base commit is available for comparison
- git fetch --no-tags --prune --depth=1 origin "$BASE_SHA" || echo "::warning::Could not fetch base SHA $BASE_SHA — all changed migrations will be shown as new"
- fi
-
- # Initialize comment body for SQL changes
- COMMENT_BODY="We've detected new migrations on this PR. Review the SQL output for each migration:\n\n"
- HAS_NEW_MIGRATIONS=false
- MIGRATION_COUNT=0
-
- # Process each changed migration file (excluding Rust migrations)
- for file in $CHANGED_FILES; do
- # Skip Rust migrations as they're handled separately by sqlx
- if [[ $file =~ rust/persons_migrations ]]; then
- continue
- fi
- # Full migration name, not the bare numeric prefix: squash
- # files (0001_squash_*) share prefixes with the historical
- # files they replace, and sqlmigrate errors on an ambiguous
- # prefix.
- if [[ $file =~ migrations/([0-9]+_[^/]+)\.py$ ]]; then
- migration_number="${BASH_REMATCH[1]}"
- # Get app name by looking at the directory structure
- # For new structure products/user_interviews/backend/migrations, we want user_interviews
- # For old structure products/user_interviews/migrations, we want user_interviews
- if [[ $file =~ products/([^/]+)/backend/migrations/ ]]; then
- app_name="${BASH_REMATCH[1]}"
- else
- app_name=$(echo $file | sed -E 's|^([^/]+/)*([^/]+)/migrations/.*|\2|')
- fi
-
- # A stacked PR can carry master's migration at this path. Keep a
- # conflicting version visible even when master has the same path.
- if git cat-file -e "$BASE_SHA:$file" 2>/dev/null || git diff --quiet origin/master HEAD -- "$file"; then
- echo "Skipping $file (already exists on the base branch or matches master)"
- continue
- fi
-
- # Generated squash files carry replaces= lists thousands
- # of names long; their SQL dump is unreviewable and the
- # state replay per file adds up across an entire squash
- # PR. Detect by content, not by name, so the check can't
- # go stale.
- if grep -q "replaces = \[" "$file"; then
- echo "Skipping $file (squash migration)"
- continue
- fi
-
- HAS_NEW_MIGRATIONS=true
- MIGRATION_COUNT=$((MIGRATION_COUNT + 1))
- echo "Checking migration $migration_number for app $app_name"
-
- # Get SQL output. A sqlmigrate failure degrades to a
- # placeholder — this comment is informational, and the
- # real migrate step below still gates the job.
- if ! SQL_OUTPUT=$(python manage.py sqlmigrate $app_name $migration_number 2>&1); then
- echo "::warning::sqlmigrate failed for $app_name $migration_number — see job log"
- SQL_OUTPUT="-- sqlmigrate failed for this migration (see the Validate migrations job log)"
- fi
-
- # Add to comment body
- COMMENT_BODY+="#### [\`$file\`](https://github.com/${{ github.repository }}/blob/${{ github.sha }}/$file)\n\`\`\`sql\n$SQL_OUTPUT\n\`\`\`\n\n"
- fi
- done
-
- if [ "$HAS_NEW_MIGRATIONS" = false ]; then
- echo "No new migrations to show (all changed files already exist on base branch)"
- echo "No new Django migrations need SQL review." > /tmp/django-migration-sql.md
- post_migration_section sql clear ok "no new migrations" /tmp/django-migration-sql.md
- exit 0
- fi
-
- # Add timestamp and commit SHA to SQL changes
- TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
- COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
- COMMIT_SHORT="${COMMIT_SHA:0:7}"
- COMMENT_BODY+="\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"
-
- # Convert \n into actual newlines
- COMMENT_BODY=$(printf '%b' "$COMMENT_BODY")
- printf '%s' "$COMMENT_BODY" > /tmp/django-migration-sql.md
- if [ "$MIGRATION_COUNT" -eq 1 ]; then
- MIGRATION_SUMMARY="1 new migration to review"
- else
- MIGRATION_SUMMARY="$MIGRATION_COUNT new migrations to review"
- fi
- post_migration_section sql post warn "$MIGRATION_SUMMARY" /tmp/django-migration-sql.md
-
- - name: Run migration risk analysis and update CI report
- if: github.event_name == 'pull_request'
- # Step timeout: an analyzer hang fails THIS step (a failure, which
- # !cancelled() still publishes as a definitive verdict) instead of
- # running out the 20-min job timeout — a job timeout counts as a
- # cancellation, which the publish step now skips, silently stranding
- # the consumer. Normal analysis finishes in well under a minute.
- timeout-minutes: 5
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- # The PR checkout above uses the PR branch, which on unrebased PRs
- # predates this script — skip the CI report section rather than fail.
- post_migration_section() {
- if [ -f .github/scripts/post-django-migration-section.mjs ]; then
- node .github/scripts/post-django-migration-section.mjs "$@"
- else
- echo "Skipping Django migration section — script not present on this checkout"
- fi
- }
-
- # Get risk analysis for all unapplied migrations (including third-party).
- # --output-json gives downstream consumers (the check-run step
- # below, and any other tool) a stable structured shape to read.
- set +e # Don't exit immediately on error
- RISK_ANALYSIS=$(python manage.py analyze_migration_risk --fail-on-blocked --output-json migration_analysis.json 2>/dev/null)
- EXIT_CODE=$?
- set -e # Re-enable exit on error
-
- # Save analysis to file for artifact upload
- if [ -n "$RISK_ANALYSIS" ]; then
- echo "$RISK_ANALYSIS" > migration_analysis.md
- fi
-
- if [ -n "$RISK_ANALYSIS" ] && echo "$RISK_ANALYSIS" | grep -q "Summary:"; then
- # Add timestamp and commit SHA to analysis
- TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
- COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
- COMMIT_SHORT="${COMMIT_SHA:0:7}"
- RISK_COMMENT="We've analyzed your migrations for potential risks.\n\n$RISK_ANALYSIS\n\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"
- RISK_COMMENT=$(printf '%b' "$RISK_COMMENT")
- printf '%s' "$RISK_COMMENT" > /tmp/django-migration-risk.md
- if [ $EXIT_CODE -ne 0 ]; then
- post_migration_section risk post fail "blocked migration detected" /tmp/django-migration-risk.md
- else
- post_migration_section risk post ok "migration analysis complete" /tmp/django-migration-risk.md
- fi
- elif [ $EXIT_CODE -ne 0 ]; then
- echo "Migration risk analysis failed. Check the workflow logs for details." > /tmp/django-migration-risk.md
- post_migration_section risk post fail "migration analysis failed" /tmp/django-migration-risk.md
- else
- echo "No Django migrations need risk analysis." > /tmp/django-migration-risk.md
- post_migration_section risk clear ok "no migrations to analyze" /tmp/django-migration-risk.md
- fi
-
- # Fail the job if there were blocked migrations
- if [ $EXIT_CODE -ne 0 ]; then
- exit $EXIT_CODE
- fi
-
- - name: Get app token for Migration risk check
- # Mint a posthog-tests installation token so the check is
- # attributed to that app instead of github-actions[bot]. The
- # app owns no workflow runs, so its check suite contains only
- # this check — it shows up as a standalone, deterministically
- # placed check rather than nesting under an arbitrary run in
- # the crowded github-actions suite.
- #
- # Skipped on fork PRs: the app secrets aren't exposed to
- # workflows triggered from forks, so the token can't be minted
- # there. The analyzer still runs in earlier steps and fails the
- # job on Blocked migrations; reviewers can read the verdict in
- # the uploaded migration-analysis artifact.
- #
- # continue-on-error: publishing the check is auxiliary. A
- # transient token-mint failure (GitHub API blip, secret
- # rotation, app misconfig) must not redden check-migrations
- # when migration validation itself passed. On failure the
- # token output is empty and the publish step below is skipped.
- #
- # !cancelled(): a cancelled run was superseded by a newer one that
- # publishes the verdict, so don't mint a token here. (${{ }} wraps
- # the leading !, a YAML tag indicator.)
- if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }}
- id: migration-risk-app-token
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }}
-
- - name: Publish Migration risk check
- # Migration risk classification is published as a GitHub check on
- # the head commit so it shows up in the PR UI alongside CI checks
- # and so any tool that already reads check_runs (review bots,
- # branch-protection rules, dashboards) can consume it without
- # parsing the comment. The check is a CI feature; consumers and
- # the analyzer are decoupled.
- #
- # Published via the posthog-tests app token (see the token step
- # above) so the check lands in that app's own check suite.
- #
- # Published on success or genuine failure (including an analyzer
- # crash, or zero migrations to analyze) so every PR ends up with a
- # definitive verdict on its head SHA. Consumers can then treat
- # "no completed check yet" purely as "CI hasn't finished" and
- # don't need a fallback heuristic.
- #
- # !cancelled(): a cancelled job skipped the analyzer, leaving no
- # migration_analysis.json, so an always() publish would stamp a
- # spurious "❌ Analyzer failed". The newer run that superseded it
- # publishes the real verdict; genuine crashes still publish here.
- #
- # Skipped on PRs from forks: the app token can't be minted there
- # (secrets are withheld from fork-triggered workflows), so the
- # check-runs POST would have no credential. The analyzer itself
- # still runs in earlier steps and fails the job on Blocked
- # migrations; reviewers can read the verdict in the uploaded
- # migration-analysis artifact.
- #
- # Also skipped when the token step failed: its output is then
- # empty and the check-runs POST would 401 and redden the job
- # for a reason unrelated to migration validation.
- if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && steps.migration-risk-app-token.outputs.token != '' }}
- env:
- GITHUB_TOKEN: ${{ steps.migration-risk-app-token.outputs.token }}
- HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- REPO: ${{ github.repository }}
- run: |
- if [ ! -f migration_analysis.json ]; then
- # Analyzer crashed before writing JSON. Publish a failure
- # check so stamphog (or any consumer) gets a definitive
- # verdict instead of looping in "wait for the check."
- echo "No migration_analysis.json — analyzer didn't run; publishing failure check"
- jq -n \
- --arg name "Migration risk" \
- --arg head_sha "$HEAD_SHA" \
- '{name: $name, head_sha: $head_sha, status: "completed",
- conclusion: "failure",
- output: {title: "❌ Analyzer failed",
- summary: "<!-- stamphog:v1 [] -->\nMigration analyzer did not produce migration_analysis.json. Re-run the Backend CI job; if it keeps failing, inspect the analyzer step logs."}}' \
- | gh api "repos/$REPO/check-runs" --method POST --input -
- exit 0
- fi
-
- MAX_LEVEL=$(jq -r '.max_level // "none"' migration_analysis.json)
-
- case "$MAX_LEVEL" in
- "Safe") CONCLUSION=success; TITLE="✅ All migrations safe" ;;
- "Needs Review") CONCLUSION=neutral; TITLE="⚠️ Needs review" ;;
- "Blocked") CONCLUSION=failure; TITLE="❌ Blocked migrations" ;;
- "none"|"null")
- # Zero Django migrations to analyze. Publish success so
- # PRs that touch only ClickHouse/async/rbac migrations
- # (which the analyzer doesn't cover) don't leave
- # consumers waiting on a check that would never come.
- CONCLUSION=success
- TITLE="✅ No Django migrations to analyze"
- ;;
- *)
- echo "Unknown max_level '$MAX_LEVEL' — publishing failure"
- CONCLUSION=failure
- TITLE="❌ Unknown analyzer output"
- ;;
- esac
-
- # Embed the analyzed file paths in the summary as a hidden
- # marker. Stamphog parses this to scope its deny-list bypass
- # to exactly the files the analyzer classified — heuristics
- # over directory names produce false bypasses for unrelated
- # systems (ClickHouse, async migrations) that share the
- # `migrations/` directory naming.
- ANALYZED_PATHS=$(jq -c '[.migrations[].file_path | select(. != null)]' migration_analysis.json 2>/dev/null || echo '[]')
- MARKER="<!-- stamphog:v1 ${ANALYZED_PATHS} -->"
-
- # Truncate the human report to fit the check-run output
- # limit (~64KB); the full markdown is also posted in the CI
- # report for humans. The marker stays at the top so it's
- # never lost to truncation.
- REPORT_BODY=$(head -c 59000 migration_analysis.md 2>/dev/null || echo "See the CI report")
- SUMMARY=$(printf '%s\n%s' "$MARKER" "$REPORT_BODY")
-
- jq -n \
- --arg name "Migration risk" \
- --arg head_sha "$HEAD_SHA" \
- --arg conclusion "$CONCLUSION" \
- --arg title "$TITLE" \
- --arg summary "$SUMMARY" \
- '{name: $name, head_sha: $head_sha, status: "completed",
- conclusion: $conclusion,
- output: {title: $title, summary: $summary}}' \
- | gh api "repos/$REPO/check-runs" --method POST --input -
-
- - name: Upload migration analysis artifact
- if: always() && github.event_name == 'pull_request'
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- with:
- name: migration-analysis
- path: |
- migration_analysis.md
- migration_analysis.json
- if-no-files-found: ignore
-
- - name: Run migrations for this PR
- run: |
- # Run Django migrations first (excluding managed=False models)
- python manage.py migrate
- # Then run persons migrations using sqlx
- DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
- sqlx migrate run --source rust/persons_migrations/
-
- - name: Replay the full migration history from scratch
- # The steps above prime the database from the cached master schema dump, so
- # the migrate that follows only applies the migrations this branch adds.
- # That hides a branch whose new migration moves an older one's position in
- # the plan, because the older one is already recorded as applied and never
- # runs against its new neighbours. A replay onto an empty database is the
- # only step that runs the whole history in the order this branch produces.
- # Master pushes already replay from scratch, which is where such a break
- # surfaces today, one merge too late.
- #
- # Queue lane only. The trunk-merge/** branch is master plus the batch, so it
- # sees the ordering every merged migration produces together; a PR branch
- # sees it only while it is current with master. The replay is by far the
- # most expensive part of this job, so paying it once per batch rather than
- # on every push of every migration branch is what keeps it affordable.
- if: github.event_name == 'pull_request' && startsWith(github.head_ref, 'trunk-merge/') && needs.changes.outputs.migration_replay == 'true'
- env:
- REPLAY_DB: posthog_migration_replay
- run: |
- set -euo pipefail
- docker compose -f docker-compose.dev.yml exec -T db psql -q -U posthog postgres \
- -c "DROP DATABASE IF EXISTS ${REPLAY_DB};" \
- -c "CREATE DATABASE ${REPLAY_DB};"
- if ! DATABASE_URL="postgres://posthog:posthog@localhost:5432/${REPLAY_DB}" python manage.py migrate; then
- echo "::error::The migration history does not apply to an empty database. A merge to master fails the same way. Read the traceback above for the migration that stopped, and check whether another migration this branch adds moved it in the plan."
- exit 1
- fi
-
- - name: Check migrations
- # Skip migration safety check on master push (no migration_files from path filter)
- if: github.event_name != 'push'
- run: |
- DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
- sqlx migrate info --source rust/persons_migrations/
- python manage.py makemigrations --check --dry-run
-
- - name: List this PR's ClickHouse migrations
- env:
- BASE_SHA: ${{ github.event.pull_request.base.sha }}
- run: |
- set -euo pipefail
- export LC_ALL=C
- DIFF_BASE=origin/master
- if [ -n "$BASE_SHA" ]; then
- if ! git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then
- git fetch --no-tags --depth=1 --filter=blob:none origin "$BASE_SHA"
- fi
- DIFF_BASE="$BASE_SHA"
- fi
- ch_migrations() {
- # Migrations share most of their text, so rename detection would pair this PR's
- # migration with one master added later and report it as R, dropping it here.
- git diff --no-renames --name-only --diff-filter="$1" "$2"..HEAD -- 'posthog/clickhouse/migrations/[0-9]*' | sort
- }
- ch_migrations A origin/master > "$RUNNER_TEMP/ch-master-added.txt"
- ch_migrations AM origin/master > "$RUNNER_TEMP/ch-master-changed.txt"
- # A stacked branch can carry master's migrations, and master can edit one later.
- # Only a file that master lacks counts as this PR's addition.
- ch_migrations A "$DIFF_BASE" | comm -12 - "$RUNNER_TEMP/ch-master-added.txt" > "$RUNNER_TEMP/ch-migrations-added.txt"
- ch_migrations AM "$DIFF_BASE" | comm -12 - "$RUNNER_TEMP/ch-master-changed.txt" > "$RUNNER_TEMP/ch-migrations-changed.txt"
- cat "$RUNNER_TEMP/ch-migrations-changed.txt"
-
- - name: Check CH migrations
- run: python manage.py test_ch_migrations_are_safe < "$RUNNER_TEMP/ch-migrations-added.txt"
-
- - name: Render CH migration SQL per cloud environment
- # CH migrations can build different `operations` lists at import time depending on
- # `settings.CLOUD_DEPLOYMENT` (e.g. cloud-only Kafka tables). Render the SQL each
- # environment will execute so reviewers can spot per-env divergence and verify gated
- # branches don't sneak in ON CLUSTER or other forbidden patterns. We render once per
- # environment and reuse the output both for the Actions log groups and the CI report
- # (environment -> node type(s) -> SQL). Environments that render identically are
- # grouped together; if all environments match, the environment level is collapsed.
- id: render_ch_sql
- if: github.event_name == 'pull_request'
- run: |
- CHANGED=$(cat "$RUNNER_TEMP/ch-migrations-changed.txt")
- if [ -z "$CHANGED" ]; then
- echo "No ClickHouse migrations changed."
- echo "has_changes=false" >> "$GITHUB_OUTPUT"
- exit 0
- fi
- echo "has_changes=true" >> "$GITHUB_OUTPUT"
- echo "migration_count=$(echo "$CHANGED" | grep -c .)" >> "$GITHUB_OUTPUT"
- {
- echo '<!-- ch-migration-sql -->'
- echo '## ClickHouse migration SQL per cloud environment'
- echo ''
- } > ch_migration_sql_comment.md
- mkdir -p ch_sql_env
- DEPLOYMENTS=('' US EU DEV)
- LABELS=(unset US EU DEV)
- # Bucket environments by identical rendered output (parallel indexed arrays;
- # avoids bash 4 associative arrays) so divergent SQL prints once.
- BUCKET_HASH=()
- BUCKET_LABEL=()
- BUCKET_FILE=()
- for idx in "${!DEPLOYMENTS[@]}"; do
- file="ch_sql_env/${idx}.md"
- # Render once per environment; the same output feeds both the log group and the comment.
- # On failure (e.g. an import error in the migration) write a visible marker rather than
- # silently dropping the environment — stderr still shows in the step log for debugging.
- if ! CLOUD_DEPLOYMENT="${DEPLOYMENTS[$idx]}" python manage.py print_ch_migration_sql --format markdown ${CHANGED} > "$file"; then
- echo '- _⚠️ failed to render SQL for this environment — see the workflow logs_' > "$file"
- fi
- echo "::group::CLOUD_DEPLOYMENT=${LABELS[$idx]}"
- cat "$file"
- echo "::endgroup::"
- h=$(sha1sum "$file" | cut -d' ' -f1)
- found=-1
- for b in "${!BUCKET_HASH[@]}"; do
- if [ "${BUCKET_HASH[$b]}" = "$h" ]; then found=$b; break; fi
- done
- if [ "$found" -ge 0 ]; then
- BUCKET_LABEL[$found]="${BUCKET_LABEL[$found]}, ${LABELS[$idx]}"
- else
- BUCKET_HASH+=("$h")
- BUCKET_LABEL+=("${LABELS[$idx]}")
- BUCKET_FILE+=("$file")
- fi
- done
- if [ "${#BUCKET_HASH[@]}" -eq 1 ]; then
- # Every environment renders the same SQL — collapse the environment level.
- echo '_Identical across all cloud environments (unset, US, EU, DEV)._' >> ch_migration_sql_comment.md
- echo '' >> ch_migration_sql_comment.md
- cat "${BUCKET_FILE[0]}" >> ch_migration_sql_comment.md
- else
- for b in "${!BUCKET_HASH[@]}"; do
- echo "- **${BUCKET_LABEL[$b]}**" >> ch_migration_sql_comment.md
- sed 's/^/ /' "${BUCKET_FILE[$b]}" >> ch_migration_sql_comment.md
- done
- fi
-
- - name: Post CH migration SQL to CI report comment
- # Sticky section in the shared CI report comment so pushes don't spam the PR.
- # GITHUB_TOKEN is read-only for fork PRs, so only comment on same-repo PRs.
- # Skipped on trunk-merge/** branches: merge-queue draft PRs are ephemeral and
- # unread, and each section write spends shared GITHUB_TOKEN budget.
- if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !startsWith(github.head_ref, 'trunk-merge/')
- env:
- HAS_CHANGES: ${{ steps.render_ch_sql.outputs.has_changes }}
- CH_MIGRATION_COUNT: ${{ steps.render_ch_sql.outputs.migration_count }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- if [ -f .github/scripts/post-ch-migration-section.mjs ]; then
- node .github/scripts/post-ch-migration-section.mjs
- node frontend/bin/ci-report/delete-legacy-comments.mjs
- else
- echo "Skipping CH migration section — script not present on this checkout"
- fi
-
- check-openapi-types:
- needs: [changes]
- if: (needs.changes.outputs.backend == 'true' || needs.changes.outputs.openapi_types == 'true')
- timeout-minutes: 20
- outputs:
- deterministic_failure: ${{ steps.deterministic-failure.outputs.deterministic_failure || 'false' }}
- permissions:
- contents: read
-
- name: Validate OpenAPI types
- runs-on: depot-ubuntu-24.04
- env:
- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- # For same-repo PRs, checkout the actual branch (not the merge commit)
- # so generated OpenAPI types can be committed directly. Fork PRs fall
- # back to the default merge commit (auto-commit bails out for forks).
- ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') && github.event.pull_request.head.ref || github.ref }}
- clean: false
-
- - name: Clean up data directories with container permissions
- run: |
- # Use docker to clean up files created by containers
- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
- continue-on-error: true
-
- - name: Log in to Docker Hub
- continue-on-error: true
- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
- with:
- username: ${{ vars.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
- - name: Stop/Start stack with Docker Compose
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- run: |
- bin/ci-wait-for-docker launch --background --down db clickhouse
-
- - name: Mint setup-action GitHub token
- id: setup-gh-token
- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}
- skip-token-revoke: true
-
- # Tool cache steps inlined rather than the setup-python-cached composite:
- # this job checks out the PR head ref, which cannot resolve a local
- # action that only exists on master until the branch rebases.
- - name: Restore Python tool cache
- id: python-toolcache
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: ${{ runner.tool_cache }}/Python/3.14.7
- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-3.14.7
-
- - name: Set up Python
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: 3.14.7
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- - name: Save Python tool cache
- if: github.ref == 'refs/heads/master' && steps.python-toolcache.outputs.cache-hit != 'true'
- uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: ${{ runner.tool_cache }}/Python/3.14.7
- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-3.14.7
-
- # Direct install: Generated changes target the PR head, which may predate the local action.
- - name: Install uv
- id: setup-uv
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- with:
- version: '0.13.4'
- enable-cache: true
- cache-dependency-glob: uv.lock
- save-cache: ${{ github.ref == 'refs/heads/master' }}
-
- - name: Install SAML (python3-saml) dependencies
- if: steps.setup-uv.outputs.cache-hit != 'true'
- run: |
- sudo rm -f /etc/apt/sources.list.d/*twingate*
- sudo apt-get update
- sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-
- - name: Install python dependencies
- run: |
- UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
-
- - name: Install pnpm dependencies
- uses: ./.github/actions/pnpm-install
- with:
- install-command: pnpm --filter=@posthog/root --filter=@posthog/frontend... --filter=@posthog/mcp... install --frozen-lockfile
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- # Installs only the desktop API client generator, not the Electron app. The desktop
- # workspace defaults to the hoisted linker, which ignores --filter and installs every
- # package, so the isolated linker is forced here. A branch that predates the generator
- # move skips this step.
- - name: Install desktop API client generator
- working-directory: products/desktop
- run: |
- if [ ! -f ../../bin/build-desktop-api-client.sh ]; then
- echo "This branch predates the desktop API client regen, so there is nothing to install."
- exit 0
- fi
- pnpm install --frozen-lockfile --filter @posthog/api-client --ignore-scripts --config.node-linker=isolated
-
- - name: Wait for Docker services
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- run: bin/ci-wait-for-docker wait --only db clickhouse
-
- - name: Add OpenAPI Problem Matcher
- run: echo "::add-matcher::.github/openapi-problem-matcher.json"
-
- - name: Check and update OpenAPI types
- id: openapi-check
- env:
- EVENT_NAME: ${{ github.event_name }}
- HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
- BRANCH: ${{ github.event.pull_request.head.ref }}
- HEAD_SHA: ${{ github.event.pull_request.head.sha }}
- run: |
- if ! ./bin/hogli build:openapi; then
- echo ""
- echo "::error::OpenAPI schema generation failed."
- echo ""
- echo "If the failure above mentions \"enum naming encountered a non-optimally"
- echo "resolvable collision\" or \"SchemaGenerationError: Failing as requested due"
- echo "to warnings\", drf-spectacular found a problem in a serializer/viewset and"
- echo "\`--fail-on-warn\` turned it into an error."
- echo ""
- echo "Diagnose locally:"
- echo " python manage.py find_enum_collisions # prints a suggested override entry to add"
- echo " hogli build:openapi-schema # to reproduce the full warning set"
- echo ""
- echo "Where to fix:"
- echo " posthog/settings/web.py # ENUM_NAME_OVERRIDES (see comment block at the top)"
- echo " /improving-drf-endpoints # invoke this skill for the full serializer/viewset guide"
- echo ""
- exit 1
- fi
- pnpm --filter=@posthog/mcp run scaffold-yaml -- --sync-all
- if git diff --exit-code; then
- echo "OpenAPI types are up to date"
- echo "needs-commit=false" >> $GITHUB_OUTPUT
- exit 0
- fi
-
- echo "OpenAPI types are out of date"
-
- # A push to a merge queue branch ends the queue attempt.
- if [[ "$BRANCH" == trunk-merge/* ]]; then
- echo "::error::OpenAPI types are out of date once this pull request is merged with master: $(git diff --name-only | tr '\n' ' ')"
- echo "Merge master into the pull request branch and push. CI regenerates the types on that branch."
- exit 1
- fi
-
- # On non-PR builds or fork PRs, fail with instructions
- if [ "$EVENT_NAME" != "pull_request" ] || \
- [ "$HEAD_REPO" != "PostHog/posthog" ]; then
- echo ""
- echo "::error::OpenAPI types are out of date!"
- echo ""
- echo "The TypeScript API types in products/*/frontend/generated/ are auto-generated"
- echo "from Django serializers and views. When you modify the backend API, you need"
- echo "to regenerate these types."
- echo ""
- echo "To fix, run locally: hogli build:openapi"
- echo "Then commit the updated generated files."
- echo ""
- echo "More info: https://posthog.com/handbook/engineering/type-system"
- echo ""
- echo "Questions? #team-devex on Slack"
- exit 1
- fi
-
- echo "::notice::Committing updated OpenAPI types to PR branch"
-
- # Verify branch hasn't advanced since CI started
- CURRENT_SHA=$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1)
- if [ "$CURRENT_SHA" != "$HEAD_SHA" ]; then
- echo "::error::Branch advanced during workflow ($HEAD_SHA -> $CURRENT_SHA) — cannot auto-commit OpenAPI types."
- echo ""
- echo "OpenAPI types are out of date and could not be auto-committed because the branch"
- echo "was updated while this workflow was running. Please run locally and push:"
- echo ""
- echo " hogli build:openapi"
- echo ""
- echo "Then commit the updated generated files."
- echo "needs-commit=false" >> $GITHUB_OUTPUT
- exit 1
- fi
-
- # Commit exactly the files this check flagged. A hand-listed pattern
- # silently dropped generated paths it did not cover, so the PR passed on
- # an empty commit and every later master push failed this same check.
- DRIFTED_FILES=$(git diff --name-only | tr '\n' ' ')
- echo "Committing: $DRIFTED_FILES"
- echo "drifted-files=$DRIFTED_FILES" >> $GITHUB_OUTPUT
- echo "needs-commit=true" >> $GITHUB_OUTPUT
-
- # Deterministic = the OpenAPI check step itself failed (not a transient
- # setup or commit-step failure), so a retry can't fix it. Mirrors repo-checks.
- - name: Flag deterministic failure
- id: deterministic-failure
- if: failure() && steps.openapi-check.outcome == 'failure'
- run: echo "deterministic_failure=true" >> "$GITHUB_OUTPUT"
-
- - name: Get app token for OpenAPI type commits
- if: steps.openapi-check.outputs.needs-commit == 'true'
- id: openapi-app-token
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }}
-
- - name: Disable auto-merge before OpenAPI type commit
- if: steps.openapi-check.outputs.needs-commit == 'true'
- env:
- GH_TOKEN: ${{ steps.openapi-app-token.outputs.token }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- run: gh pr merge --disable-auto "$PR_NUMBER" || echo "Auto-merge was not enabled"
-
- - name: Commit OpenAPI types via GitHub API (signed)
- if: steps.openapi-check.outputs.needs-commit == 'true'
- uses: planetscale/ghcommit-action@25309d8005ac7c3bcd61d3fe19b69e0fe47dbdde # v0.2.20
- with:
- commit_message: 'chore: update OpenAPI generated types'
- repo: ${{ github.repository }}
- branch: ${{ github.event.pull_request.head.ref }}
- file_pattern: ${{ steps.openapi-check.outputs.drifted-files }}
- env:
- GITHUB_TOKEN: ${{ steps.openapi-app-token.outputs.token }}
-
- cancel-backend-on-openapi-check-failure:
- needs: [check-openapi-types]
- if: >
- always() &&
- needs.check-openapi-types.outputs.deterministic_failure == 'true' &&
- github.event_name == 'pull_request' &&
- github.event.pull_request.head.repo.full_name == github.repository
- timeout-minutes: 5
- permissions:
- contents: read
- actions: write
- name: Cancel Backend CI after OpenAPI check failure
- runs-on: ubuntu-24.04
- steps:
- - name: Cancel run
- env:
- GH_TOKEN: ${{ github.token }}
- GH_REPO: ${{ github.repository }}
- run: |
- echo "::notice title=OpenAPI check failed deterministically::Canceling remaining Backend CI jobs. A retry cannot fix this failure; push a fix to start a new run."
- gh run cancel ${{ github.run_id }}
-
- build_django_matrix:
- name: Build Django matrix
- needs: [changes, get_clickhouse_versions, turbo-discover]
- if: |
- !cancelled() &&
- needs.changes.outputs.backend == 'true' &&
- needs.get_clickhouse_versions.result == 'success'
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 5
- outputs:
- include: ${{ steps.build.outputs.include }}
- steps:
- - name: Build matrix include list
- id: build
- env:
- OLDEST_SUPPORTED_IMAGE: ${{ needs.get_clickhouse_versions.outputs.oldest_supported_image }}
- COMPAT_MATRIX_JSON: ${{ needs.get_clickhouse_versions.outputs.compat_matrix }}
- DJANGO_SHARDS_JSON: ${{ needs.turbo-discover.outputs.django_shards }}
- MODE: ${{ needs.turbo-discover.outputs.mode }}
- RUN_POE: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').run_poe }}
- RUN_TEMPORAL: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').run_temporal }}
- CORE_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').core_files }}
- COMPAT_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').compat_files }}
- SEGMENT_SHARDS: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').segment_shards }}
- # A trunk-merge/** PR opens as a draft but is the merge gate, and
- # turbo-discover deliberately does not select for it. Reporting it as a
- # draft here would turn that empty MODE into "skip" and build an
- # empty matrix, so the queue would merge having run no Django tests
- # at all. Not-draft makes the empty MODE fall through to the full
- # matrix, which is what the queue has to run.
- IS_DRAFT: ${{ github.event.pull_request.draft && !startsWith(github.head_ref, 'trunk-merge/') }}
- IS_PR_RUN: ${{ github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') }}
- FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'run-ci-backend') }}
- # The new-events-schema variants double the matrix, so they run only on
- # opted-in PRs and manual dispatches. Every other run takes the
- # events_json rows, which cover .github/new-events-schema-targets.txt only.
- RUN_NEW_EVENTS_SCHEMA: ${{ contains(github.event.pull_request.labels.*.name, 'test-new-events-schema') || github.event_name == 'workflow_dispatch' }}
- # Narrowed events_json paths and their shard count. Empty files on a run
- # that takes the whole list.
- JSON_TARGETS_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_files }}
- JSON_TARGETS_SHARDS: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_shards }}
- # Same rule as the django job's condition: the suite runs for a legacy
- # edit, for a product change that reaches legacy code, and when discovery
- # failed, which leaves run_legacy empty. Without it only the events_json
- # rows for the product matrix run.
- DJANGO_SUITE: ${{ needs.changes.outputs.legacy == 'true' || needs.turbo-discover.outputs.run_legacy != 'false' }}
- run: |
- # :NOTE: Keep shard counts/group ranges in sync with historical Django matrix tuning.
- # Consult #team-devex before changing.
-
- # A draft PR must never fall back to the full matrix. turbo-discover sets
- # MODE to "skip"/"full"/"selected"; an empty MODE on a
- # draft means it was cancelled or failed (typically ready_for_review
- # superseding it mid-flight). Without this, the draft builds the full
- # matrix and holds the per-branch concurrency slot for the whole run, so the
- # ready-for-review run — the actual merge gate — queues behind it instead
- # of starting. Skip here and defer to that ready run. The run-ci-backend
- # label intentionally forces the full matrix on a draft, so honor it.
- if [[ -z "$MODE" && "$IS_DRAFT" == "true" && "$FORCE_FULL" != "true" ]]; then
- echo "::notice::Draft PR with no trusted Django selection — skipping heavy matrices; the ready-for-review run is the full gate"
- MODE="skip"
- fi
-
- if [[ "$MODE" == "skip" ]]; then
- # Draft PR where selection couldn't be trusted: skip the heavy
- # matrices and defer to the full run on ready for review.
- echo "include=[]" >> "$GITHUB_OUTPUT"
- echo "Django matrix size: 0 (mode=skip)"
- exit 0
- fi
-
- if [[ "$DJANGO_SUITE" != "true" ]]; then
- # A product change that reaches no legacy code skips the Django suite.
- # The product jobs read the legacy events table only, so the events_json
- # rows still run the listed paths of the products in the product matrix.
- core="[]"
- core_new_events_schema="[]"
- core_persons_on_events="[]"
- core_persons_on_events_new_schema="[]"
- temporal="[]"
- temporal_new_events_schema="[]"
- compat="[]"
- json_targets_count=0
- [[ -n "$JSON_TARGETS_FILES" ]] && json_targets_count="${JSON_TARGETS_SHARDS:-1}"
- elif [[ "$MODE" == "selected" ]]; then
- # Selected mode: run only the segments that have selected files. Shard
- # counts come from turbo-discover, which applies its per-shard budget to
- # the selected tests' recorded seconds. A wide selection therefore
- # spreads over several jobs instead of running to the job timeout in
- # one. No counts means one shard per segment.
- segment_shards="${SEGMENT_SHARDS:-}"
- [[ -z "$segment_shards" ]] && segment_shards='{"core": 1, "poe": 1, "temporal": 1}'
-
- core_count=0
- [[ -n "$CORE_FILES" ]] && core_count=$(jq -r '.core' <<< "$segment_shards")
- core=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$core_count" '
- [range(1; $n + 1) | {
- segment: "Core",
- "person-on-events": false,
- "new-events-schema": false,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $n,
- group: .,
- artifact_key: ("core-" + (.|tostring)),
- compat: false
- }]
- ')
- core_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$core_count" '
- [range(1; $n + 1) | {
- segment: "Core",
- "person-on-events": false,
- "new-events-schema": true,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $n,
- group: .,
- artifact_key: ("core-events-json-" + (.|tostring)),
- compat: false
- }]
- ')
- poe_count=0
- [[ "$IS_PR_RUN" != "true" && "$RUN_POE" == "true" ]] && poe_count=$(jq -r '.poe' <<< "$segment_shards")
- core_persons_on_events=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$poe_count" '
- [range(1; $n + 1) | {
- segment: "Core",
- "person-on-events": true,
- "new-events-schema": false,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $n,
- group: .,
- artifact_key: ("core-poe-" + (.|tostring)),
- compat: false
- }]
- ')
- core_persons_on_events_new_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$poe_count" '
- [range(1; $n + 1) | {
- segment: "Core",
- "person-on-events": true,
- "new-events-schema": true,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $n,
- group: .,
- artifact_key: ("core-poe-events-json-" + (.|tostring)),
- compat: false
- }]
- ')
- temporal_count=0
- [[ "$RUN_TEMPORAL" == "true" ]] && temporal_count=$(jq -r '.temporal' <<< "$segment_shards")
- temporal=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$temporal_count" '
- [range(1; $n + 1) | {
- segment: "Temporal",
- "person-on-events": false,
- "new-events-schema": false,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $n,
- group: .,
- artifact_key: ("temporal-" + (.|tostring)),
- compat: false
- }]
- ')
- temporal_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$temporal_count" '
- [range(1; $n + 1) | {
- segment: "Temporal",
- "person-on-events": false,
- "new-events-schema": true,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $n,
- group: .,
- artifact_key: ("temporal-events-json-" + (.|tostring)),
- compat: false
- }]
- ')
- # Compat covers the older supported ClickHouse versions over
- # CLICKHOUSE_COMPAT_PYTEST_TARGETS. Selection can drop it only when the
- # diff touches none of those paths — otherwise a narrowed run would
- # silently lose the one signal the full matrix has about old servers.
- # One unsharded entry per version, over the selected compat files.
- if [[ -n "$COMPAT_FILES" ]]; then
- compat=$(jq -cn --argjson compat "${COMPAT_MATRIX_JSON:-[]}" '
- [ $compat
- | map(."clickhouse-server-image") | unique | to_entries[]
- | {
- "clickhouse-server-image": .value,
- segment: "Core",
- "person-on-events": false,
- "new-events-schema": false,
- "python-version": "3.14.7",
- concurrency: 1,
- group: 1,
- compat: true,
- artifact_key: ("compat-" + ((.key + 1)|tostring))
- }
- ]
- ')
- else
- compat="[]"
- fi
- # The narrowed events_json paths: selected files under a listed path,
- # plus the listed paths of the products in the narrowed product matrix.
- json_targets_count=0
- [[ -n "$JSON_TARGETS_FILES" ]] && json_targets_count="${JSON_TARGETS_SHARDS:-1}"
- else
- # Full run: auto-shard counts from turbo-discover (Amdahl's law on .test_durations).
- # Falls back to hardcoded defaults if turbo-discover failed or data is missing.
- CORE_SHARDS=38
- CORE_POE_SHARDS=7
- TEMPORAL_SHARDS=7
- JSON_TARGETS_FULL_SHARDS=5
- if [ -n "$DJANGO_SHARDS_JSON" ] && echo "$DJANGO_SHARDS_JSON" | jq -e '.Core.shards' > /dev/null 2>&1; then
- CORE_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r '.Core.shards')
- CORE_POE_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r '.CorePOE.shards')
- TEMPORAL_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r '.Temporal.shards')
- JSON_TARGETS_FULL_SHARDS=$(echo "$DJANGO_SHARDS_JSON" | jq -r --argjson fallback "$JSON_TARGETS_FULL_SHARDS" '.JsonTargets.shards // $fallback')
- echo "Auto-sharding: Core=$CORE_SHARDS, CorePOE=$CORE_POE_SHARDS, Temporal=$TEMPORAL_SHARDS, JsonTargets=$JSON_TARGETS_FULL_SHARDS"
- else
- echo "::warning::Django shard auto-sizing unavailable, using defaults: Core=$CORE_SHARDS, CorePOE=$CORE_POE_SHARDS, Temporal=$TEMPORAL_SHARDS, JsonTargets=$JSON_TARGETS_FULL_SHARDS"
- fi
- # A full run takes every listed events_json path, unless discovery
- # shortened the list: a skipped product, or a product-only diff whose
- # legacy paths the merge queue runs. An empty list gives 0 rows.
- json_targets_count="$JSON_TARGETS_FULL_SHARDS"
- if [[ -n "$JSON_TARGETS_SHARDS" ]] && (( JSON_TARGETS_SHARDS < json_targets_count )); then
- json_targets_count="$JSON_TARGETS_SHARDS"
- echo "::notice::Discovery shortened the events_json list to $json_targets_count shard(s)"
- fi
-
- # The legacy-mode safeguard leg runs only where a regression must not
- # land: merge queue and master. Ordinary PR lanes run the
- # customer-default mode only, so a PR pays no shards for the legacy
- # mode. The queue's trunk-merge/** run is a pull_request event as well,
- # so it is exempted by branch. Compat keeps its own legacy-mode rows in
- # every lane, because it guards old servers, not this.
- if [[ "$IS_PR_RUN" == "true" ]]; then
- CORE_POE_SHARDS=0
- fi
-
- core=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_SHARDS" '
- [range(1; $shards + 1) | {
- segment: "Core",
- "person-on-events": false,
- "new-events-schema": false,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $shards,
- group: .,
- artifact_key: ("core-" + (.|tostring)),
- compat: false
- }]
- ')
-
- core_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_SHARDS" '
- [range(1; $shards + 1) | {
- segment: "Core",
- "person-on-events": false,
- "new-events-schema": true,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $shards,
- group: .,
- artifact_key: ("core-events-json-" + (.|tostring)),
- compat: false
- }]
- ')
-
- core_persons_on_events=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_POE_SHARDS" '
- [range(1; $shards + 1) | {
- segment: "Core",
- "person-on-events": true,
- "new-events-schema": false,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $shards,
- group: .,
- artifact_key: ("core-poe-" + (.|tostring)),
- compat: false
- }]
- ')
-
- core_persons_on_events_new_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$CORE_POE_SHARDS" '
- [range(1; $shards + 1) | {
- segment: "Core",
- "person-on-events": true,
- "new-events-schema": true,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $shards,
- group: .,
- artifact_key: ("core-poe-events-json-" + (.|tostring)),
- compat: false
- }]
- ')
-
- temporal=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$TEMPORAL_SHARDS" '
- [range(1; $shards + 1) | {
- segment: "Temporal",
- "person-on-events": false,
- "new-events-schema": false,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $shards,
- group: .,
- artifact_key: ("temporal-" + (.|tostring)),
- compat: false
- }]
- ')
-
- temporal_new_events_schema=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson shards "$TEMPORAL_SHARDS" '
- [range(1; $shards + 1) | {
- segment: "Temporal",
- "person-on-events": false,
- "new-events-schema": true,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $shards,
- group: .,
- artifact_key: ("temporal-events-json-" + (.|tostring)),
- compat: false
- }]
- ')
-
- compat_source="${COMPAT_MATRIX_JSON:-[]}"
- compat=$(jq -cn --argjson compat "$compat_source" '
- [
- $compat
- | to_entries[]
- | .value + {
- segment: "Core",
- "person-on-events": false,
- "new-events-schema": false,
- "python-version": "3.14.7",
- compat: true,
- artifact_key: ("compat-" + ((.key + 1)|tostring))
- }
- ]
- ')
- fi
-
- if [[ "$RUN_NEW_EVENTS_SCHEMA" != "true" ]]; then
- # Without the schema copies, only the events_json rows below refresh
- # *.new_events_schema.ambr files, so a module outside
- # .github/new-events-schema-targets.txt lets its file go stale.
- echo "Skipping new-events-schema variants (RUN_NEW_EVENTS_SCHEMA is off)"
- core_new_events_schema="[]"
- core_persons_on_events_new_schema="[]"
- temporal_new_events_schema="[]"
- else
- # The doubled matrices exclude some curated paths, such as Dagster tests.
- # Discovery keeps only those paths so the other tests do not run a third time.
- json_targets_count="${JSON_TARGETS_SHARDS:-0}"
- fi
-
- # The events_json rows: Core jobs over the listed paths with the
- # native-JSON events table. They keep Core's person-on-events default.
- json_targets=$(jq -cn --arg image "$OLDEST_SUPPORTED_IMAGE" --argjson n "$json_targets_count" '
- [range(1; $n + 1) | {
- segment: "Core",
- "json-targets": true,
- "person-on-events": false,
- "new-events-schema": true,
- "python-version": "3.14.7",
- "clickhouse-server-image": $image,
- concurrency: $n,
- group: .,
- artifact_key: ("json-targets-" + (.|tostring)),
- compat: false
- }]
- ')
-
- include=$(jq -cn \
- --argjson core "$core" \
- --argjson core_new_events_schema "$core_new_events_schema" \
- --argjson core_persons_on_events "$core_persons_on_events" \
- --argjson core_persons_on_events_new_schema "$core_persons_on_events_new_schema" \
- --argjson temporal "$temporal" \
- --argjson temporal_new_events_schema "$temporal_new_events_schema" \
- --argjson compat "$compat" \
- --argjson json_targets "$json_targets" \
- '$core + $core_new_events_schema + $core_persons_on_events + $core_persons_on_events_new_schema + $temporal + $temporal_new_events_schema + $compat + $json_targets')
-
- echo "include=$include" >> "$GITHUB_OUTPUT"
- echo "Django matrix size: $(jq -r 'length' <<< "$include") (mode=$MODE)"
-
- django:
- needs: [changes, turbo-discover, detect-snapshot-mode, get_clickhouse_versions, build_django_matrix]
- # Run legacy pytest if:
- # 1. Legacy code directly changed (ee/, posthog/)
- # 2. OR product changes affect legacy code (turbo-discover outputs run_legacy=true)
- # 3. OR turbo-discover itself failed (conservative: run Django on detection failure)
- # 4. OR a product-only change has events_json paths to run. build_django_matrix
- # then builds only the events_json rows, because the Django suite does not run.
- # The non-empty include guard covers drafts in skip mode and selected runs
- # that chose zero files — an empty matrix would otherwise fail the job.
- # A status function is required to inspect dependency results after failures.
- # Use !cancelled(), not always(), so superseded matrix jobs still receive cancellation.
- if: |
- !cancelled() &&
- needs.changes.outputs.backend == 'true' &&
- needs.build_django_matrix.result == 'success' &&
- needs.build_django_matrix.outputs.include != '[]' &&
- (needs.changes.outputs.legacy == 'true' ||
- needs.turbo-discover.outputs.run_legacy == 'true' ||
- (needs.turbo-discover.result != 'success' && needs.turbo-discover.result != 'skipped') ||
- fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_files != '')
- # increase for tmate testing
- # 45 for slow Temporal shards on master; keep <= deploy gate wait (container-images-cd.yml)
- timeout-minutes: 45
-
- name: Django tests – ${{ matrix.segment }}${{ matrix.compat && ' compat' || '' }}${{ matrix.json-targets && ' json-targets' || '' }} (persons-on-events ${{ (matrix.segment == 'Core' && !matrix.person-on-events && !matrix.compat) && 'on' || 'off' }}, events schema ${{ matrix.new-events-schema && 'json' || 'legacy' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
- # Runner type is performance-critical — consult #team-devex before changing
- runs-on: depot-ubuntu-24.04
-
- strategy:
- fail-fast: false
- matrix:
- include: ${{ fromJson(needs.build_django_matrix.outputs.include) }}
- env:
- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-
- steps:
- # The default ref, which is the PR merge commit. turbo-discover selects the test files
- # from this tree, so a checkout of any other ref can lack a file that the selection lists.
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- fetch-depth: 1
- lfs: true
- clean: false
-
- # Put signal-fanout on PATH so the pytest steps below can use it as
- # their `shell:` wrapper. GHA's `shell:` field doesn't allow
- # ${{ github.workspace }} expansion, so a plain name on PATH is the
- # only portable way to reference it.
- - name: Install signal-fanout
- shell: bash
- run: sudo install -m 0755 .github/scripts/signal-fanout /usr/local/bin/signal-fanout
-
- - name: Clean up data directories with container permissions
- run: |
- # Use docker to clean up files created by containers
- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
- continue-on-error: true
-
- - name: 'Safeguard: ensure no stray Python modules at product root'
- run: |
- echo "Checking that products/* only contain backend/, frontend/, or shared/ as Python code roots..."
- BAD_FILES=$(find products -maxdepth 2 -type f -name "*.py" ! -path "*/backend/*" ! -name "__init__.py" ! -name "conftest.py" -o -maxdepth 2 -type d -name "migrations" ! -path "*/backend/*")
- if [ -n "$BAD_FILES" ]; then
- echo "❌ Found Python code or migrations outside backend/:"
- echo "$BAD_FILES"
- echo "Please move these into the appropriate backend/ folder."
- exit 1
- fi
- echo "✅ No stray Python files or migrations found at product roots."
-
- # Pre-tests
-
- # Copies the fully versioned UDF xml file for use in CI testing
- - name: Log in to Docker Hub
- continue-on-error: true
- if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
- with:
- username: ${{ vars.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
- - name: Stop/Start stack with Docker Compose
- shell: bash
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- COMPOSE_PROFILES: temporal,azure
- CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image || needs.get_clickhouse_versions.outputs.oldest_supported_image }}
- WAIT_FOR_DOCKER_LAUNCH_RETRY_DELAY: 5
- run: |
- cp posthog/user_scripts/latest_user_defined_function.xml docker/clickhouse/user_defined_function.xml
- bin/ci-wait-for-docker launch --background --down-all-profiles
-
- - name: Add service hostnames to /etc/hosts
- shell: bash
- run: echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts
-
- - name: Mint setup-action GitHub token
- id: setup-gh-token
- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ vars.GH_APP_POSTHOG_DEVEX_GENERAL_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_DEVEX_GENERAL_PRIVATE_KEY }}
- skip-token-revoke: true
-
- # Restore before setup-python so the tool cache is warm when it starts.
- - name: Restore Python tool cache
- id: python-toolcache
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: ${{ runner.tool_cache }}/Python/${{ matrix.python-version }}
- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}
-
- # Sequential, not a `parallel:` block. Both of these write $GITHUB_PATH,
- # and concurrent writes crash the GitHub Actions runner ("Collection was
- # modified" or a missing-key error). The runner fails the step even though
- # the tool installed fine, so the job dies in setup with nothing to debug.
- - name: Set up Python
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version: ${{ matrix.python-version }}
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- - name: Install Rust
- if: needs.changes.outputs.backend == 'true'
- uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
- with:
- toolchain: 1.91.1
- components: cargo
-
- - name: Save Python tool cache
- if: github.ref == 'refs/heads/master' && steps.python-toolcache.outputs.cache-hit != 'true'
- uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: ${{ runner.tool_cache }}/Python/${{ matrix.python-version }}
- key: python-toolcache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}
-
- # Direct install: Tests run the explicit PR head, which may predate the local action.
- - name: Install uv
- id: setup-uv-tests
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- with:
- version: '0.13.4'
- enable-cache: true
- cache-dependency-glob: uv.lock
- save-cache: ${{ github.ref == 'refs/heads/master' }}
-
- - name: Install SAML (python3-saml) dependencies
- if: ${{ needs.changes.outputs.backend == 'true' && steps.setup-uv-tests.outputs.cache-hit != 'true' }}
- shell: bash
- run: |
- sudo rm -f /etc/apt/sources.list.d/*twingate* && sudo apt-get update && sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-
- - name: Install sqlx-cli
- if: needs.changes.outputs.backend == 'true'
- uses: ./.github/actions/setup-sqlx-cli
-
- - name: Determine if hogql-parser has changed compared to master
- shell: bash
- id: hogql-parser-diff
- run: |
- git fetch --no-tags --prune --depth=1 origin master
- changed=$(git diff --quiet HEAD origin/master -- common/hogql_parser/ && echo "false" || echo "true")
- echo "changed=$changed" >> $GITHUB_OUTPUT
- # A branch that predates the crate has nothing to install.
- rust_changed=false
- if [[ -f rust/hogql/parser/pyproject.toml ]]; then
- crate_version=$(grep -m1 '^version = ' rust/hogql/parser/pyproject.toml | sed -E 's/version = "(.*)"/\1/')
- # Publishing is a manual step after merge, so master can carry a crate version that
- # the root pin does not name yet. That version builds from source too.
- if ! git diff --quiet HEAD origin/master -- rust/hogql/parser/ || ! grep -q "\"hogql-parser-rs==$crate_version\"" pyproject.toml; then
- rust_changed=true
- fi
- fi
- echo "rust_changed=$rust_changed" >> $GITHUB_OUTPUT
-
- # tests would intermittently fail in GH actions
- # with exit code 134 _after passing_ all tests
- # this appears to fix it
- # absolute wild tbh https://stackoverflow.com/a/75503402
-
- # Cache Qt library packages to reduce network dependency and flakiness
- # Using cache-apt-pkgs-action instead of setup-qt-libs to avoid network issues
- - name: Cache and install Qt libraries
- uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
- with:
- packages: libegl1 libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 x11-utils libxcb-cursor0 libopengl0
- version: 1.0
-
- - name: Install Python dependencies
- shell: bash
- run: |
- UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
-
- - name: Set up Node.js for canvas builder
- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && hashFiles('products/canvas/packages/canvas_builder/package.json') != '' }}
- uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
- with:
- node-version-file: .nvmrc
- token: ${{ steps.setup-gh-token.outputs.token || github.token }}
-
- - name: Install canvas builder dependencies
- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}
- shell: bash
- run: |
- if [[ -f products/canvas/packages/canvas_builder/package.json ]]; then
- npm ci --ignore-scripts --omit=dev --no-audit --no-fund --prefix products/canvas/packages/canvas_builder
- fi
-
- - name: Install the working version of hogql-parser
- if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.changed == 'true' }}
- shell: bash
- # This is not cached, because the current HEAD version of hogql-parser must be built if it has
- # changed (requirements.txt has the already-published version)
- run: |
- # Short budget: the `||` mirror is the real fallback for a host that is down.
- fetch=(curl --fail --location --retry 3 --retry-all-errors --retry-max-time 30 --connect-timeout 10)
- "${fetch[@]}" https://www.antlr.org/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip \
- || "${fetch[@]}" https://raw.githubusercontent.com/antlr/website-antlr4/gh-pages/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip
- # Check that the downloaded archive is the expected runtime - a security measure
- anltr_known_md5sum="c875c148991aacd043f733827644a76f"
- antlr_found_ms5sum="$(md5sum antlr4-source.zip | cut -d' ' -f1)"
- if [[ "$anltr_known_md5sum" != "$antlr_found_ms5sum" ]]; then
- echo "Unexpected MD5 sum of antlr4-source.zip!"
- echo "Known: $anltr_known_md5sum"
- echo "Found: $antlr_found_ms5sum"
- exit 64
- fi
- unzip antlr4-source.zip -d antlr4-source && cd antlr4-source
- cmake .
- DESTDIR=out make install
- sudo cp -r out/usr/local/include/antlr4-runtime /usr/include/
- sudo cp out/usr/local/lib/libantlr4-runtime.so* /usr/lib/
- sudo ldconfig
- cd ..
- pip install ./common/hogql_parser
-
- - name: Install the working version of hogql-parser-rs
- if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.rust_changed == 'true' }}
- shell: bash
- # rust-py is the production parser, so its tests must run against this branch's
- # crate rather than the version pinned in the root pyproject.toml.
- run: pip install ./rust/hogql/parser
-
- - name: Set up needed files
- shell: bash
- run: |
- mkdir -p frontend/dist
- touch frontend/dist/index.html
- touch frontend/dist/layout.html
- touch frontend/dist/exporter.html
- ./bin/download-mmdb
-
- - name: Wait for Docker services
- shell: bash
- # Read-only health poll for core services only. Can't use --wait here
- # because it blocks on ALL project containers including temporal, which
- # boots slowly (auto-setup runs DB migrations). Temporal was started in
- # background above and will be ready by the time temporal tests run.
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- COMPOSE_PROFILES: temporal,azure
- CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image || needs.get_clickhouse_versions.outputs.oldest_supported_image }}
- run: bin/ci-wait-for-docker wait
-
- - name: Restore schema cache from master
- if: ${{ github.event_name == 'pull_request' && needs.turbo-discover.outputs.schema_cache_key != '' }}
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: schema.sql.gz
- # Prefer the shared content key; fall back to the exact merge-base SHA key,
- # then to the newest dump of any migration set. Right after a migration
- # lands on master both exact keys miss until ci-backend re-dumps, and a
- # from-scratch migrate costs ~17 minutes per shard; pytest's keepdb
- # migrate tops a slightly stale dump up in seconds instead.
- key: ${{ needs.turbo-discover.outputs.schema_migrations_key || needs.turbo-discover.outputs.schema_cache_key }}
- restore-keys: |
- ${{ needs.turbo-discover.outputs.schema_cache_key }}
- ${{ needs.turbo-discover.outputs.schema_prefix_key }}
-
- - name: Prime test_posthog from cached schema
- # Treat cache problems as misses; the migrate step below rebuilds the DB
- # whenever this one reports restored=false.
- id: prime-schema
- if: ${{ github.event_name == 'pull_request' }}
- run: |
- echo "restored=false" >> "$GITHUB_OUTPUT"
- if [ ! -f schema.sql.gz ]; then
- echo "::notice::Schema cache miss; the migrate step will build test_posthog"
- exit 0
- fi
- mkdir -p .postgres-backups
- mv schema.sql.gz .postgres-backups/schema-latest.sql.gz
- if ./bin/hogli db:restore-test-db; then
- echo "restored=true" >> "$GITHUB_OUTPUT"
- else
- echo "::warning::Schema restore failed (stale/incompatible cached dump?); the migrate step will build test_posthog"
- fi
-
- - name: Migrate test_posthog from scratch
- # Any run without a restored schema cache (master, dispatches, PR cache
- # misses) replays the full migration chain here, in a process that exits
- # before tests: the executor's memory high-water stays resident in
- # whichever process runs it, and pytest must not carry it through the
- # test phase. pytest --reuse-db adopts the DB like the PR prime above;
- # any failure falls back to the in-pytest migrate.
- if: ${{ github.event_name != 'pull_request' || steps.prime-schema.outputs.restored != 'true' }}
- env:
- DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog
- run: |
- admin_psql() { docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres "$@"; }
- if ! admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" -c "CREATE DATABASE test_posthog;"; then
- echo "::warning::Could not recreate test_posthog; pytest --reuse-db will run the full migrate itself"
- exit 0
- fi
- if ! python manage.py migrate; then
- echo "::warning::Out-of-process migrate failed; pytest --reuse-db will run the full migrate itself"
- admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" || true
- fi
-
- - name: Restore test durations from cache
- # pytest-split distributes Core/Temporal tests across shards using
- # .test_durations. Restore the newest cache; the fallback step below covers
- # a miss.
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: .test_durations
- # run_id never matches — forces the prefix restore-key to the newest entry.
- key: posthog-test-durations-${{ github.run_id }}
- restore-keys: |
- posthog-test-durations-
-
- - name: Fall back to the last published durations
- # .test_durations is not in the repo. The timing workflow refreshes the
- # cache several times a day and publishes the same files as an artifact
- # of its own runs. On a cache miss (retention lapsed, timing workflow
- # down for two weeks) take the newest artifact; if that is gone too,
- # pytest-split splits by count and turbo-discover sizes from file counts
- # until the next timing run.
- if: ${{ hashFiles('.test_durations') == '' }}
- continue-on-error: true
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \
- --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')
- if [ -z "$run_id" ]; then
- echo "::warning::No successful timing run to fall back to, sharding without durations"
- exit 0
- fi
- gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \
- || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"
-
- - name: Restore per-segment test durations from cache
- # File-granularity sharding plans from a per-segment file (.test_durations.core /
- # .test_durations.temporal) scoped to exactly what the segment runs, avoiding the
- # union file's cross-segment contamination. These are cache-only as well,
- # written by the daily timing-update job on master so PR runs can restore them.
- # On a miss the file is simply absent and the "Run … tests" step omits
- # --split-plan-path, falling back to scoping the union (still correct, pre-fix
- # balance). The posthog-segment-durations- prefix is deliberately distinct from
- # the union's posthog-test-durations- so the two never shadow each other.
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: |
- .test_durations.core
- .test_durations.temporal
- key: posthog-segment-durations-${{ github.run_id }}
- restore-keys: |
- posthog-segment-durations-
-
- - name: Compute hypothesis constants cache key
- # hypothesis builds its constants pool of property-test inputs by
- # AST-parsing every local module in sys.modules. It caches the result
- # in .hypothesis/constants under a hash of each source file, so the
- # pool is content-addressed: restoring a stale copy is safe, because a
- # changed file reads as a miss and rebuilds. CI otherwise pays the full
- # parse on every shard's pytest collection (about 10 s per shard).
- id: hyp-constants-key
- shell: bash
- run: |
- # The key includes the installed hypothesis version because the
- # entry format is an implementation detail, and rotates weekly so
- # entries for deleted or rewritten files expire. The restore prefix
- # covers the gap until the first master run of a new week saves.
- hyp_version=$(python -c "import hypothesis; print(hypothesis.__version__)")
- prefix="posthog-hypothesis-constants-${hyp_version}-${HYPOTHESIS_CONSTANTS_EPOCH}-"
- {
- echo "key=${prefix}$(date -u +%G-%V)"
- echo "restore_prefix=${prefix}"
- } >> "$GITHUB_OUTPUT"
-
- - name: Restore hypothesis constants cache
- id: hyp-constants
- uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- # Constants only. Do NOT add .hypothesis/examples: the example
- # database decides which inputs property tests replay, so reusing
- # one across runs changes what the tests check.
- path: .hypothesis/constants
- key: ${{ steps.hyp-constants-key.outputs.key }}
- restore-keys: |
- ${{ steps.hyp-constants-key.outputs.restore_prefix }}
-
- - name: Download the run's sharding plan snapshot
- # Pin every attempt to the plan turbo-discover snapshotted (rationale on
- # that step); a missing snapshot falls back to the floating caches above.
- id: pinned-plan
- continue-on-error: true
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- name: test-durations-plan
- path: /tmp/test-durations-plan
-
- - name: Apply the pinned sharding plan
- if: steps.pinned-plan.outcome == 'success'
- shell: bash
- run: |
- if [ ! -f /tmp/test-durations-plan/.test_durations ]; then
- echo "::warning::Sharding-plan snapshot is incomplete, keeping the floating cache restores"
- exit 0
- fi
- # rm first so segment files the snapshot lacks don't survive the floating restore
- rm -f .test_durations.core .test_durations.temporal
- cp /tmp/test-durations-plan/.test_durations .test_durations
- for f in .test_durations.core .test_durations.temporal events-schema-prune.json; do
- if [ -f "/tmp/test-durations-plan/$f" ]; then cp "/tmp/test-durations-plan/$f" "$f"; fi
- done
- echo "Sharding plan pinned to this run's snapshot"
-
- - name: Warn when a re-run reshards unpinned
- if: github.run_attempt != '1' && steps.pinned-plan.outcome != 'success'
- shell: bash
- run: echo "::warning::Sharding-plan snapshot unavailable, this re-run reshards from the floating timing caches and may not re-execute tests that failed in earlier attempts"
-
- - name: Determine if --snapshot-update should be on
- # UPDATE mode: human commits - update snapshots
- # CHECK mode: bot commits (after snapshot update) - verify snapshots match exactly
- # safeguard allowlist leg (matrix.person-on-events): always update, because it
- # runs a subset under the non-default mode and so does not own snapshots
- if: ${{ needs.changes.outputs.backend == 'true' && (needs.detect-snapshot-mode.outputs.mode == 'update' || matrix.person-on-events) }}
- shell: bash
- # --snapshot-warn-unused: pytest-split shards individual tests across
- # runners, so a shard reports other shards' snapshots as "unused". The flag
- # downgrades that from a failure to a warning. It does NOT prevent deletion:
- # --snapshot-update only spares a snapshot when syrupy matches it to a test
- # (by file and name) that did not run on this shard.
- run: echo "PYTEST_ARGS=--snapshot-update --snapshot-warn-unused" >> $GITHUB_ENV
-
- - name: Add snapshot flags for compat subset runs
- # Compat runs a subset and doesn't own snapshots (main Django rows do).
- # Warn on unused snapshots to avoid version-specific false negatives.
- if: ${{ needs.changes.outputs.backend == 'true' && matrix.compat }}
- shell: bash
- run: echo "PYTEST_ARGS=--snapshot-update --snapshot-warn-unused" >> $GITHUB_ENV
-
- - name: Warn (don't fail) on unused snapshots for sharded CHECK runs
- # CHECK mode (fork PRs, the 'no-snapshot-update' label) runs without
- # --snapshot-update, so the steps above don't fire and no snapshot flags
- # are set. But pytest-split shards individual tests across runners, so each
- # shard only exercises a subset of the snapshots in a given file — every
- # snapshot owned by a test that landed on another shard is reported as
- # "unused". Under sharding that signal is structurally unreliable, so without
- # --snapshot-warn-unused a shard fails on those false positives even when
- # every test that ran passed (the snapshot report's "N snapshots unused"
- # line). Every non-fork run (UPDATE, poe, compat) already passes the flag;
- # this just makes forks symmetric. Trade-off: it downgrades unused snapshots
- # to a warning, so genuinely orphaned snapshots go uncaught here — but they
- # already do everywhere else, and sharding makes per-shard orphan detection
- # impossible regardless. Snapshot *mismatches* and missing snapshots still
- # fail, so real value drift is still caught. Mutually exclusive with the
- # steps above (mode != update, not poe, not compat).
- if: ${{ needs.changes.outputs.backend == 'true' && needs.detect-snapshot-mode.outputs.mode != 'update' && !matrix.person-on-events && !matrix.compat }}
- shell: bash
- run: echo "PYTEST_ARGS=--snapshot-warn-unused" >> $GITHUB_ENV
-
- # Tests
- - name: Log test environment diagnostics
- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}
- shell: bash
- env:
- SHARD_GROUP: ${{ matrix.group }}
- SHARD_CONCURRENCY: ${{ matrix.concurrency }}
- CLICKHOUSE_IMAGE: ${{ matrix.clickhouse-server-image }}
- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- COMMIT_SHA: ${{ github.sha }}
- BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
- run: |
- echo "=== Test Environment Diagnostics ==="
- echo "Shard: $SHARD_GROUP/$SHARD_CONCURRENCY"
- echo "Python version: $(python --version)"
- echo "ClickHouse version: $CLICKHOUSE_IMAGE"
- echo "Events schema: $CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA"
- echo "Commit: $COMMIT_SHA"
- echo "Branch: $BRANCH_NAME"
- echo "Runner: $(uname -a)"
- echo "Memory: $(free -h | head -2)"
- echo "Disk: $(df -h / | tail -1)"
- echo "Docker containers:"
- docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || true
- echo "==================================="
-
- - name: Run Core tests
- id: run-core-tests
- # continue-on-error so a test failure doesn't fail the job here; the
- # quarantine gate at the end of the job is the verdict — it passes when
- # every failure is an already-quarantined flake, fails otherwise.
- continue-on-error: true
- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}
- env:
- # Person-on-events v2 reads are the production default for every
- # project created since 2024-06-14, so the full Core rows run with the
- # mode on. matrix.person-on-events marks the safeguard allowlist leg
- # that keeps the legacy joined mode covered, not the mode itself: every
- # other condition keyed on it (targets, snapshot ownership, timing
- # artifacts) selects the leg. Compat rows run the legacy mode as well,
- # as the old-server safeguard.
- # marker: person-on-events-default. The run block below greps the
- # checked-out workflow for this token to tell a head that carries the
- # flip from one that predates it.
- PERSON_ON_EVENTS_V2_ENABLED: ${{ !matrix.person-on-events && !matrix.compat && 'true' || 'false' }}
- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- MODE: ${{ needs.turbo-discover.outputs.mode }}
- CORE_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').core_files }}
- POE_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').poe_files }}
- COMPAT_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').compat_files }}
- JSON_TARGETS: ${{ matrix.json-targets && 'true' || 'false' }}
- # The narrowed events_json paths. Empty when the row takes the whole list.
- JSON_TARGETS_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').json_targets_files }}
- # devex: measure core (posthog/ee) coverage on PRs to size the gap vs products. sysmon = low overhead.
- COVERAGE_CORE: sysmon
- # backend-coverage-report skips narrowed runs (patch coverage would read
- # as false gaps), so instrumenting one just costs time and uploads an
- # artifact nothing reads. The events_json rows rerun tests the legacy
- # rows already measure, so they skip it as well.
- COV_ARGS: ${{ github.event_name == 'pull_request' && needs.turbo-discover.outputs.mode != 'selected' && !matrix.json-targets && '--cov=posthog --cov=ee --cov-config=.github/coverage-core.cfg --cov-report=' || '' }}
- # Wrap bash with signal-fanout so the runner's cancel signal is
- # propagated to the entire process tree (incl. pytest). Without
- # this, GHA only signals the top-level bash and pytest survives
- # until the runner's 10s SIGKILL — which sometimes misses
- # detached descendants entirely. Installed earlier on PATH.
- shell: 'signal-fanout bash --noprofile --norc -eo pipefail {0}'
- run: |
- set +e
- # Enable retries only when the checkout includes retry diagnostics.
- RETRY_ARGS=()
- if grep -qF '"posthog.runner_name"' posthog/conftest.py; then
- RETRY_ARGS=(--force-reruns 1 --reruns-delay 1)
- fi
- # A missing --cov-config file aborts pytest, so skip coverage if it is absent.
- if [[ -n "$COV_ARGS" && ! -f .github/coverage-core.cfg ]]; then
- COV_ARGS=""
- fi
- # A checkout without the matching test pins must use legacy mode.
- if [[ "$PERSON_ON_EVENTS_V2_ENABLED" == "true" ]] && ! grep -qF "marker: person-on-events-default" .github/workflows/ci-backend.yml; then
- echo "::notice::Checked-out revision predates the person-on-events default, running Core in legacy mode"
- export PERSON_ON_EVENTS_V2_ENABLED=false
- fi
- # Unsupported pytest-split flags abort collection, so probe the installed plugin.
- SPLITTING_ALGORITHM=duration_based_chunks
- SPLIT_GRANULARITY=()
- if python -c "from pytest_split.algorithms import Algorithms; from pytest_split.plugin import PytestSplitFilePlugin; raise SystemExit(0 if 'optimal_chunks' in Algorithms.names() else 1)" 2>/dev/null; then
- SPLITTING_ALGORITHM=optimal_chunks
- SPLIT_GRANULARITY=(--split-granularity=file)
- # Scope the file-mode plan to Core's clean per-segment durations
- # (restored from the daily master cache) when the pinned pytest-split
- # supports it, so the plan isn't poisoned by the union .test_durations's
- # cross-segment tests — see AGENTS.md (CI). Only persons-on-events-off,
- # non-compat Core matches the .core run-set; POE-on, compat, and the
- # events_json rows run a different/narrower scope, and a cache miss with
- # no committed file all fall back to scoping the union.
- if [ -f .test_durations.core ] && [[ "${{ matrix.person-on-events }}" != "true" ]] && [[ "${{ matrix.compat }}" != "true" ]] && [[ "$JSON_TARGETS" != "true" ]] && python -c "from pytest_split.plugin import SUPPORTS_SPLIT_PLAN_PATH" 2>/dev/null; then
- SPLIT_GRANULARITY+=(--split-plan-path .test_durations.core)
- fi
- fi
- if [[ "$JSON_TARGETS" == "true" ]]; then
- # The events_json leg. The job checks out the PR head, and a head without
- # the list also predates the JSON-mode test fixes that came before it, so
- # its listed tests would fail on code the PR did not touch. The merge
- # queue's trunk-merge/** head always has the list, so the gate still runs.
- if [[ ! -f .github/new-events-schema-targets.txt ]]; then
- echo "::notice::The checked-out revision predates .github/new-events-schema-targets.txt, so the events_json leg runs nothing until this branch rebases onto master"
- exit 0
- fi
- # A narrowed run hands over its paths, and every other run takes the
- # whole list. Parse it the way turbo-discover.js does.
- if [[ -n "$JSON_TARGETS_FILES" ]]; then
- read -r -a json_candidates <<< "$JSON_TARGETS_FILES"
- else
- mapfile -t json_candidates < <(sed -e 's/#.*//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's:/*$::' -e '/^$/d' .github/new-events-schema-targets.txt)
- fi
- # The PR head can lack a path that the merged list names. pytest stops
- # with a usage error on a missing path, so skip that path and say so.
- json_targets=()
- for target in "${json_candidates[@]}"; do
- if [[ -e "$target" ]]; then
- json_targets+=("$target")
- else
- echo "::warning::events_json path $target does not exist in this checkout, so it does not run"
- fi
- done
- if [[ ${#json_targets[@]} -eq 0 ]]; then
- exit 0
- fi
- JSON_SPLIT=()
- if [[ "${{ matrix.concurrency }}" -gt 1 ]]; then
- JSON_SPLIT=(--splits "${{ matrix.concurrency }}" --group "${{ matrix.group }}"
- --splitting-algorithm="$SPLITTING_ALGORITHM" "${SPLIT_GRANULARITY[@]}")
- fi
- if [[ "$GITHUB_EVENT_NAME" == "pull_request" && "$GITHUB_HEAD_REF" != trunk-merge/* && -f events-schema-prune.json ]]; then
- # Skips the tests that a recording showed never read the JSON tables. The
- # merge queue runs every listed test, so a skipped test whose code starts
- # to read those tables still fails before the change reaches master.
- export POSTHOG_EVENTS_SCHEMA_PRUNE_MANIFEST=events-schema-prune.json
- fi
- # No --store-durations, because the timing data describes legacy-table runs only.
- # shellcheck disable=SC2086
- pytest -v --tb=short --reuse-db -o junit_duration_report=call "${json_targets[@]}" -m "not async_migrations" \
- "${JSON_SPLIT[@]}" \
- "${RETRY_ARGS[@]}" -r fEsxXR \
- --junitxml=junit-core.xml \
- $COV_ARGS \
- $PYTEST_ARGS
- elif [[ "$MODE" == "selected" ]]; then
- # Selected mode: run only the test files chosen by the snob shadow selector.
- # Sharded only when build_django_matrix sized the segment above one shard,
- # no --store-durations (don't pollute timing data with a partial run).
- # One pytest retry overrides product/marker budgets; Engineering analytics
- # retains the retry count and failed-attempt diagnostics even when it passes.
- if [[ "${{ matrix.compat }}" == "true" ]]; then
- targets="$COMPAT_FILES"
- elif [[ "${{ matrix.person-on-events }}" == "true" ]]; then
- targets="$POE_FILES"
- else
- targets="$CORE_FILES"
- fi
- # A selection wide enough to need more than one shard is split like a
- # full run. Item granularity on purpose: the target list is already
- # narrow, so pre-import file skipping saves nothing, and the per-segment
- # plan file describes a different run-set than the selected subset.
- SELECTED_SPLIT=()
- if [[ "${{ matrix.concurrency }}" -gt 1 ]]; then
- SELECTED_SPLIT=(--splits "${{ matrix.concurrency }}" --group "${{ matrix.group }}"
- --splitting-algorithm="$SPLITTING_ALGORITHM")
- fi
- # shellcheck disable=SC2086
- pytest -v --tb=short --reuse-db -o junit_duration_report=call $targets -m "not async_migrations" \
- "${SELECTED_SPLIT[@]}" \
- "${RETRY_ARGS[@]}" -r fEsxXR \
- --junitxml=junit-core.xml \
- $COV_ARGS \
- $PYTEST_ARGS
- else
- full_targets="posthog ee/"
- full_ignores=()
- if [[ "${{ matrix.compat }}" == "true" ]]; then
- full_targets="$CLICKHOUSE_COMPAT_PYTEST_TARGETS"
- elif [[ "${{ matrix.person-on-events }}" == "true" ]]; then
- full_targets="./posthog/clickhouse/ ./posthog/api/test/dashboards/test_dashboard.py ee/clickhouse/"
- full_ignores+=(--ignore=posthog/hogql_queries --ignore=posthog/hogql)
- fi
- # shellcheck disable=SC2086
- pytest -v --tb=short --reuse-db -o junit_duration_report=call $full_targets -m "not async_migrations" \
- --ignore=posthog/temporal \
- --ignore=posthog/dags \
- --ignore=common/hogvm/python/test \
- --ignore=posthog/test/repo_invariants \
- "${full_ignores[@]}" \
- --splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \
- --durations=1000 --durations-min=1.0 --store-durations \
- --pytest-durations=100 \
- --splitting-algorithm="$SPLITTING_ALGORITHM" \
- "${SPLIT_GRANULARITY[@]}" \
- "${RETRY_ARGS[@]}" -r fEsxXR \
- --junitxml=junit-core.xml \
- $COV_ARGS \
- $PYTEST_ARGS
- fi
- exit_code=$?
- set -e
- if [ $exit_code -eq 5 ]; then
- echo "No tests collected for this shard, this is expected when splitting tests"
- exit 0
- else
- exit $exit_code
- fi
-
- - name: Upload core coverage
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- if: ${{ always() && github.event_name == 'pull_request' && needs.turbo-discover.outputs.mode != 'selected' && matrix.segment == 'Core' && !matrix.json-targets }}
- with:
- name: coverage-core-${{ strategy.job-index }}
- path: .coverage
- include-hidden-files: true
- if-no-files-found: ignore
- retention-days: 2
- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwise
-
- # Uncomment this code to create an ssh-able console so you can debug issues with github actions
- # (Consider changing the timeout in ci-backend.yml to have more time)
- # - name: Setup tmate session
- # if: failure()
- # uses: mxschmitt/action-tmate@v3
-
- - name: Log test environment diagnostics
- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}
- shell: bash
- env:
- SHARD_GROUP: ${{ matrix.group }}
- SHARD_CONCURRENCY: ${{ matrix.concurrency }}
- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- COMMIT_SHA: ${{ github.sha }}
- BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
- run: |
- echo "=== Test Environment Diagnostics ==="
- echo "Shard: $SHARD_GROUP/$SHARD_CONCURRENCY"
- echo "Python version: $(python --version)"
- echo "Events schema: $CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA"
- echo "Commit: $COMMIT_SHA"
- echo "Branch: $BRANCH_NAME"
- echo "Runner: $(uname -a)"
- echo "Memory: $(free -h | head -2)"
- echo "Disk: $(df -h / | tail -1)"
- echo "Docker containers:"
- docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || true
- echo "==================================="
-
- - name: Run Temporal tests
- id: run-temporal-tests
- # continue-on-error: the quarantine gate at the end of the job is the verdict.
- # See "Run Core tests".
- continue-on-error: true
- if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}
- # See "Run Core tests" for why signal-fanout wraps bash here.
- shell: 'signal-fanout bash --noprofile --norc -eo pipefail {0}'
- env:
- AWS_S3_ALLOW_UNSAFE_RENAME: 'true'
- MODAL_TOKEN_ID: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_ID || '' }}
- MODAL_TOKEN_SECRET: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_SECRET || '' }}
- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- MODE: ${{ needs.turbo-discover.outputs.mode }}
- TEMPORAL_FILES: ${{ fromJSON(needs.turbo-discover.outputs.selection || '{}').temporal_files }}
- run: |
- set +e
- # Open PR heads must have retry diagnostics before blanket retries are enabled.
- RETRY_ARGS=()
- if grep -qF '"posthog.runner_name"' posthog/conftest.py; then
- RETRY_ARGS=(--force-reruns 1 --reruns-delay 1)
- fi
- # Pick the split algorithm and granularity at runtime: optimal_chunks +
- # file granularity only when the installed pytest-split offers them, else
- # duration_based_chunks / item. A workflow edit runs merged with master, so
- # an unrebased PR on older pytest-split must not hit an unknown
- # --splitting-algorithm / --split-granularity. See AGENTS.md (CI).
- SPLITTING_ALGORITHM=duration_based_chunks
- SPLIT_GRANULARITY=()
- if python -c "from pytest_split.algorithms import Algorithms; from pytest_split.plugin import PytestSplitFilePlugin; raise SystemExit(0 if 'optimal_chunks' in Algorithms.names() else 1)" 2>/dev/null; then
- SPLITTING_ALGORITHM=optimal_chunks
- SPLIT_GRANULARITY=(--split-granularity=file)
- # No --split-plan-path here for now: the cached plan describes
- # master's Temporal run-set, which still contains the batch-exports
- # and tasks temporal suites this branch moved to their product jobs.
- # Cuts drawn around those phantom files starve two shards and
- # overload the third (observed 4/6/20 min on a 3-shard split).
- # Scoping the union instead intersects with what actually collects.
- # Restore the plan path after the timing workflow has rebuilt
- # .test_durations.temporal from post-merge junit.
- :
- fi
- if [[ "$MODE" == "selected" ]]; then
- # Selected mode: no --store-durations (don't pollute timing data with a
- # partial run). The same single-retry budget applies to selected and full runs.
- # A selection wide enough to need more than one shard is split like a
- # full run. Item granularity on purpose: the target list is already
- # narrow, so pre-import file skipping saves nothing, and the per-segment
- # plan file describes a different run-set than the selected subset.
- SELECTED_SPLIT=()
- if [[ "${{ matrix.concurrency }}" -gt 1 ]]; then
- SELECTED_SPLIT=(--splits "${{ matrix.concurrency }}" --group "${{ matrix.group }}"
- --splitting-algorithm="$SPLITTING_ALGORITHM")
- fi
- # shellcheck disable=SC2086
- pytest -v --tb=short --reuse-db -o junit_duration_report=call $TEMPORAL_FILES -m "not async_migrations" \
- "${SELECTED_SPLIT[@]}" \
- "${RETRY_ARGS[@]}" -r fEsxXR \
- --junitxml=junit-temporal.xml \
- $PYTEST_ARGS
- else
- # No per-test pytest --timeout; rely on the job-level timeout-minutes
- # as the single safety net. The per-test guillotine added nothing
- # over it (--timeout-method=thread os._exit()s the whole process
- # anyway, so the shard dies either way) and was the reason the
- # transaction=True overhead in async tests looked like "the
- # timeout problem" instead of a per-test cost worth fixing.
- pytest -v --tb=short --reuse-db -o junit_duration_report=call posthog/temporal products/signals/backend/emission -m "not async_migrations" \
- --splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \
- --durations=100 --durations-min=1.0 --store-durations \
- --pytest-durations=100 \
- --splitting-algorithm="$SPLITTING_ALGORITHM" \
- "${SPLIT_GRANULARITY[@]}" \
- "${RETRY_ARGS[@]}" -r fEsxXR \
- --junitxml=junit-temporal.xml \
- $PYTEST_ARGS
- fi
- exit_code=$?
- set -e
- if [ $exit_code -eq 5 ]; then
- echo "No tests collected for this shard, this is expected when splitting tests"
- exit 0
- else
- exit $exit_code
- fi
-
- - name: Save hypothesis constants cache
- # One Core shard per master run writes the weekly entry: a save from
- # every shard would race on the same key and churn the shared cache
- # budget. cache-hit != 'true' skips the save once this week's entry
- # exists, which keeps later master runs cheap.
- # continue-on-error: overlapping master runs at week rollover fail its
- # reservation check when an earlier run has saved the same key, and a
- # cache miss must never red a green run.
- continue-on-error: true
- if: |
- github.ref == 'refs/heads/master' &&
- matrix.segment == 'Core' &&
- matrix.group == 1 &&
- !matrix.person-on-events &&
- !matrix.new-events-schema &&
- !matrix.compat &&
- steps.hyp-constants.outputs.cache-hit != 'true'
- uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
- with:
- path: .hypothesis/constants
- key: ${{ steps.hyp-constants-key.outputs.key }}
-
- # Post tests
- - name: Show docker compose logs on failure
- if: failure() && (needs.changes.outputs.backend == 'true' && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure')
- shell: bash
- run: docker compose -f docker-compose.dev.yml logs
-
- - name: Upload updated timing data as artifacts
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- # Upload on master, or on PRs that change timing/sharding scripts (for validation)
- if: ${{ (github.ref == 'refs/heads/master' || needs.changes.outputs.timing_scripts == 'true') && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.new-events-schema && matrix.clickhouse-server-image == needs.get_clickhouse_versions.outputs.oldest_supported_image }}
- with:
- name: timing_data-${{ matrix.segment }}-${{ matrix.group }}
- path: .test_durations
- include-hidden-files: true
- retention-days: 2
-
- - name: Verify new snapshots for flakiness
- # Only in UPDATE mode - CHECK mode doesn't update snapshots
- if: ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.compat && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure' }}
- shell: bash
- # The reruns must write the same schema mode's snapshot file as the main
- # run, or the new-schema file removed before each rerun never comes back.
- env:
- CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- run: |
- .github/scripts/verify-new-snapshots.sh
-
- - name: Generate snapshot patch
- # Only in UPDATE mode - CHECK mode verifies snapshots match exactly
- if: ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.compat && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure' }}
- shell: bash
- env:
- NEW_EVENTS_SCHEMA: ${{ matrix.new-events-schema && 'true' || 'false' }}
- run: |
- mkdir -p /tmp/patches
- # Stage any new/modified .ambr files so they appear in git diff
- git add -N '*.ambr' || true
- # Each schema leg may only ship its own snapshot files. A leg still
- # *claims* the other schema mode's snapshot names without writing them,
- # so syrupy deletes them from the working tree as unused; scoping the
- # patch keeps those deletions out of the committed diff.
- if [[ "$NEW_EVENTS_SCHEMA" == "true" ]]; then
- ambr_pathspec=('*.new_events_schema.ambr')
- else
- ambr_pathspec=('*.ambr' ':(exclude)*.new_events_schema.ambr')
- fi
- # Generate patch if there are changes
- if ! git diff --quiet -- "${ambr_pathspec[@]}" 2>/dev/null; then
- git diff --binary --full-index -- "${ambr_pathspec[@]}" > /tmp/patches/backend-${{ matrix.artifact_key }}.patch
- echo "Generated patch with $(wc -l < /tmp/patches/backend-${{ matrix.artifact_key }}.patch) lines"
- else
- echo "No snapshot changes to patch"
- fi
-
- - name: Upload snapshot patch
- # Only in UPDATE mode
- if: ${{ needs.detect-snapshot-mode.outputs.mode == 'update' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events && !matrix.compat && steps.run-core-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure' }}
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- with:
- name: snapshot-patch-${{ matrix.artifact_key }}
- path: /tmp/patches/
- if-no-files-found: ignore
- retention-days: 1
-
- - name: Archive email renders
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- if: needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && !matrix.person-on-events && !matrix.new-events-schema && !matrix.compat
- with:
- name: email_renders-${{ github.sha }}-${{ github.run_attempt }}-${{ matrix.segment }}-${{ matrix.person-on-events }}-${{ matrix.group }}
- path: posthog/tasks/test/__emails__
- retention-days: 1
- - name: Upload test results
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- if: always()
- with:
- # Re-run attempts suffix the name: a same-name upload supersedes attempt 1's
- # artifact, and the timing reporter needs both to pair a re-run recovery
- # (attempt-N pass) with the attempt-1 failure it proves flaky.
- name: junit-results-backend-${{ matrix.artifact_key }}${{ github.run_attempt != '1' && format('-attempt{0}', github.run_attempt) || '' }}
- path: |
- junit-*.xml
- !junit-*-retry-failures.xml
-
- # The helper sets TRUNK_CODEOWNERS_PATH so Trunk attributes uploaded tests to their owners.
- - name: Build the Trunk ownership map
- if: ${{ !cancelled() && needs.changes.outputs.backend == 'true' && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}
- shell: bash
- run: |
- script=.github/scripts/trunk-codeowners.sh
- if [ -x "$script" ]; then "$script" "$RUNNER_TEMP/trunk-codeowners" "junit-*.xml"; fi
-
- # Best-effort Trunk upload (continue-on-error); the "Fail on test failure" step below is
- # the verdict, so a Trunk outage can't red a passing shard. Internal PRs only (needs the
- # secret); cli-version pinned, not 'latest' (skips the release redirect).
- # TRUNK_UPLOAD_ENABLED is the master kill-switch: when it is not 'true' this gate
- # skips, so nothing uploads and no known flakes are quarantined. TRUNK_QUARANTINE_ENABLED
- # splits the two — it defaults off, so set it to 'true' to let the verdict step below mask
- # known flakes; otherwise any real failure reds the job (quarantining needs upload on too).
- # This inlined gate lacks the shared action's retry.
- - name: Quarantine gate (Core)
- id: quarantine_gate_core
- continue-on-error: true
- if: ${{ !cancelled() && matrix.segment == 'Core' && needs.changes.outputs.backend == 'true' && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}
- uses: trunk-io/analytics-uploader@385f1ccdf345b4532dc4b6c665dd432b702b8e28 # v2.1.2
- env:
- TRUNK_HIDE_TEST_COLLECTION_LINKS: 'true'
- with:
- junit-paths: junit-core.xml,junit-core-retry-failures.xml
- org-slug: posthog-inc
- test-collection-id: AA6dvTdI # backend-core
- # Non-default rows upload under their own variant, so a failure only that mode
- # hits does not quarantine the default test with the same id.
- variant: ${{ matrix.compat && format('compat-{0}', matrix.clickhouse-server-image) || matrix.person-on-events && (matrix.new-events-schema && 'poe-off-events-json' || 'poe-off') || matrix.new-events-schema && 'events-json' || '' }}
- cli-version: '0.15.4'
- quarantine: true
- previous-step-outcome: ${{ steps.run-core-tests.outcome }}
- token: ${{ secrets.TRUNK_API_TOKEN }}
- - name: Quarantine gate (Temporal)
- id: quarantine_gate_temporal
- continue-on-error: true
- if: ${{ !cancelled() && matrix.segment == 'Temporal' && needs.changes.outputs.backend == 'true' && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}
- uses: trunk-io/analytics-uploader@385f1ccdf345b4532dc4b6c665dd432b702b8e28 # v2.1.2
- env:
- TRUNK_HIDE_TEST_COLLECTION_LINKS: 'true'
- with:
- junit-paths: junit-temporal.xml,junit-temporal-retry-failures.xml
- org-slug: posthog-inc
- test-collection-id: smyQjxZ3 # backend-temporal
- # A native-JSON row uploads under its own variant, so a failure in that mode
- # does not read as a flake of the legacy test with the same id.
- variant: ${{ matrix.new-events-schema && 'events-json' || '' }}
- cli-version: '0.15.4'
- quarantine: true
- previous-step-outcome: ${{ steps.run-temporal-tests.outcome }}
- token: ${{ secrets.TRUNK_API_TOKEN }}
- # Verdict: red a real failure the gate didn't clear as a quarantined flake. != 'success'
- # also covers the skipped gate on fork/Dependabot. Passing shards never red here.
- - name: Fail on test failure
- if: ${{ !cancelled() && ((steps.run-core-tests.outcome == 'failure' && (vars.TRUNK_QUARANTINE_ENABLED != 'true' || steps.quarantine_gate_core.outcome != 'success')) || (steps.run-temporal-tests.outcome == 'failure' && (vars.TRUNK_QUARANTINE_ENABLED != 'true' || steps.quarantine_gate_temporal.outcome != 'success'))) }}
- shell: bash
- run: exit 1
-
- # Aggregate and commit snapshot changes from all matrix jobs
-
- get_clickhouse_versions:
- name: Get ClickHouse versions
- needs: [changes]
- # Only feeds the test matrices, which master pushes skip.
- if: needs.changes.outputs.backend == 'true' && github.event_name != 'push'
- runs-on: depot-ubuntu-24.04
- timeout-minutes: 5
- outputs:
- # Oldest supported version as plain string (for comparisons)
- oldest_supported_image: ${{ steps.read-versions.outputs.oldest_supported_image }}
- # Local dev / hobby image, mirrored from clickhouse-versions.json `local`
- local_image: ${{ steps.read-versions.outputs.local_image }}
- # Fully expanded compat matrix (version x shard group)
- compat_matrix: ${{ steps.read-versions.outputs.compat_matrix }}
- steps:
- # Merge ref: turbo-tests and django check out the merge ref and mount its config.xml.
- # Reading the version from another ref would split an atomic version+config bump.
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- sparse-checkout: .github/clickhouse-versions.json
- sparse-checkout-cone-mode: false
- - name: Read ClickHouse versions from JSON
- id: read-versions
- run: |
- compat_shards="2"
- echo "Using $compat_shards shard(s) per compat version"
-
- # Oldest supported version for main Django tests (for max compatibility)
- oldest_supported=$(jq -r '.oldest_supported' .github/clickhouse-versions.json)
- if [ -z "$oldest_supported" ] || [ "$oldest_supported" = "null" ]; then
- echo "::error::No oldest_supported version found in .github/clickhouse-versions.json"
- exit 1
- fi
- echo "oldest_supported_image=$oldest_supported" >> $GITHUB_OUTPUT
- echo "Oldest supported version for Django tests: $oldest_supported"
-
- # Local dev / hobby image — the version product (turbo) tests run against.
- local_image=$(jq -r '.local' .github/clickhouse-versions.json)
- if [ -z "$local_image" ] || [ "$local_image" = "null" ]; then
- echo "::error::No local version found in .github/clickhouse-versions.json"
- exit 1
- fi
- echo "local_image=$local_image" >> $GITHUB_OUTPUT
- echo "Local dev/hobby image for product tests: $local_image"
-
- # Read all unique versions so we can derive compat coverage.
- all=$(jq -c '[.[]] | unique' .github/clickhouse-versions.json)
- if [ "$all" = "[]" ] || [ -z "$all" ]; then
- echo "::error::No versions found in .github/clickhouse-versions.json"
- exit 1
- fi
- echo "All CH versions found: $all"
-
- # Compat coverage is only needed for non-oldest versions.
- compat_versions=$(jq -c --arg oldest "$oldest_supported" '[.[]] | unique | map(select(. != $oldest))' .github/clickhouse-versions.json)
- echo "Compat versions (excluding oldest): $compat_versions"
-
- compat_count=$(jq -r 'length' <<< "$compat_versions")
- echo "compat_count=$compat_count" >> $GITHUB_OUTPUT
-
- if [ "$compat_count" -eq 0 ]; then
- echo "compat_matrix=[]" >> $GITHUB_OUTPUT
- echo "No non-oldest CH versions found — compat tests provide no additional coverage, skipping"
- else
- compat_matrix=$(jq -cn --argjson versions "$compat_versions" --argjson shards "$compat_shards" '
- [
- $versions[] as $version
- | range(1; $shards + 1) as $group
- | {
- "clickhouse-server-image": $version,
- "concurrency": $shards,
- "group": $group
- }
- ]
- ')
- echo "compat_matrix=$compat_matrix" >> $GITHUB_OUTPUT
- echo "Compat matrix: $compat_matrix"
- fi
- # Aggregate and commit snapshot changes from all matrix jobs
- handle-snapshots:
- name: Commit snapshot changes
- needs: [changes, detect-snapshot-mode, django, turbo-tests]
- # Only in UPDATE mode - CHECK mode verifies snapshots match exactly
- # Run even if some matrix jobs failed (to commit snapshots from passing jobs)
- if: ${{ !cancelled() && needs.detect-snapshot-mode.outputs.mode == 'update' && needs.changes.outputs.backend == 'true' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' }}
- runs-on: ubuntu-24.04
- timeout-minutes: 5
- permissions:
- contents: write
- pull-requests: write
- steps:
- # Look for patches before the checkout. Most runs have none, and the required
- # gate waits for this job.
- - name: Download all snapshot patches
- id: download-patches
- continue-on-error: true
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- pattern: snapshot-patch-*
- path: /tmp/snapshot-patches
- merge-multiple: true
-
- - name: Check for patches
- id: check-patches
- run: |
- if [ "${{ steps.download-patches.outcome }}" == "failure" ] || [ ! -d /tmp/snapshot-patches ]; then
- echo "has-patches=false" >> $GITHUB_OUTPUT
- echo "No snapshot patches found"
- exit 0
- fi
-
- if [ -n "$(find /tmp/snapshot-patches -name '*.patch' -type f -size +0c 2>/dev/null)" ]; then
- echo "has-patches=true" >> $GITHUB_OUTPUT
- echo "Found patches:"
- ls -la /tmp/snapshot-patches/
- else
- echo "has-patches=false" >> $GITHUB_OUTPUT
- echo "Patch files empty or missing - no snapshot changes"
- fi
-
- # Use GitHub app token so Actions run after commiting updated snapshots
- - name: Get app token
- if: steps.check-patches.outputs.has-patches == 'true'
- id: app-token
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }}
- private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }}
-
- - name: Checkout
- if: steps.check-patches.outputs.has-patches == 'true'
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- ref: ${{ github.event.pull_request.head.sha }}
- repository: ${{ github.event.pull_request.head.repo.full_name }}
- token: ${{ steps.app-token.outputs.token }}
- fetch-depth: 1
-
- - name: Require snapshot conflict checks
- if: steps.check-patches.outputs.has-patches == 'true'
- run: |
- if ! grep -Fq 'marker: snapshot-conflict-exit' .github/actions/commit-snapshots/action.yml; then
- echo "::error::Rebase onto master before committing snapshots. This PR head cannot reject conflicting snapshot patches."
- exit 1
- fi
-
- - name: Commit snapshots
- if: steps.check-patches.outputs.has-patches == 'true'
- uses: ./.github/actions/commit-snapshots
- with:
- workflow-type: backend
- patch-path: /tmp/snapshot-patches
- snapshot-path: '.'
- commit-message: 'test(backend): update query snapshots'
- pr-number: ${{ github.event.pull_request.number }}
- repository: ${{ github.repository }}
- commit-sha: ${{ github.event.pull_request.head.sha }}
- branch-name: ${{ github.event.pull_request.head.ref }}
- github-token: ${{ steps.app-token.outputs.token }}
-
- # Job just to collate the status of the matrix jobs for requiring passing status
- # Must depend on handle-snapshots to prevent auto-merge before commits complete
- django_tests:
- needs:
- [
- changes,
- get_clickhouse_versions,
- build_django_matrix,
- django,
- check-migrations,
- check-openapi-types,
- turbo-discover,
- build-product-test-matrix,
- turbo-tests,
- detect-snapshot-mode,
- handle-snapshots,
- repo-checks,
- sdk-major-guard,
- ]
- name: Django Tests Pass
- runs-on: ubuntu-24.04
- permissions:
- contents: read
- actions: read # the failure rollup downloads each shard's JUnit artifact
- # Headroom for downloading every shard's JUnit for the failure rollup.
- timeout-minutes: 8
- # Intentional !cancelled(): the gate inspects failed dependencies and fails
- # closed on live runs; a superseded run records cancelled, not failure.
- # The deterministic-failure disjuncts cover this workflow cancelling its
- # own run: without them a real repo or OpenAPI failure would report
- # cancelled too, and the failure-rate metric would lose it.
- if: >
- !cancelled()
- || needs.repo-checks.outputs.deterministic_failure == 'true'
- || needs.check-openapi-types.outputs.deterministic_failure == 'true'
- steps:
- # Kept on pull requests so the ruleset entry for this job stays satisfied until the
- # Depot gate replaces it there.
- - name: Point at the Depot gate
- if: github.event_name == 'pull_request'
- run: echo "::notice::Depot CI runs the backend tests for pull requests. The required check is 'Backend CI on Depot / Test results'."
- - name: Summarize dependency results
- if: github.event_name != 'pull_request'
- run: |
- {
- echo "### Backend CI dependency results"
- echo "| Dependency | Result |"
- echo "| --- | --- |"
- echo "| repo-checks | ${{ needs.repo-checks.result }} |"
- echo "| sdk-major-guard | ${{ needs.sdk-major-guard.result }} |"
- echo "| check-openapi-types | ${{ needs.check-openapi-types.result }} |"
- echo "| changes | ${{ needs.changes.result }} |"
- echo "| get_clickhouse_versions | ${{ needs.get_clickhouse_versions.result }} |"
- echo "| build_django_matrix | ${{ needs.build_django_matrix.result }} |"
- echo "| django | ${{ needs.django.result }} |"
- echo "| check-migrations | ${{ needs.check-migrations.result }} |"
- echo "| turbo-discover | ${{ needs.turbo-discover.result }} |"
- echo "| build-product-test-matrix | ${{ needs.build-product-test-matrix.result }} |"
- echo "| turbo-tests | ${{ needs.turbo-tests.result }} |"
- echo "| handle-snapshots | ${{ needs.handle-snapshots.result }} |"
- if [[ "${{ needs.repo-checks.outputs.deterministic_failure }}" == "true" ]]; then
- echo ""
- echo "**Root cause:** Repo checks failed deterministically. Open the repo checks job logs; downstream cancelled jobs are expected."
- fi
- if [[ "${{ needs.check-openapi-types.outputs.deterministic_failure }}" == "true" ]]; then
- echo ""
- echo "**Root cause:** OpenAPI type checks failed deterministically. Open the OpenAPI type check job logs; downstream cancelled jobs are expected."
- fi
- } | tee -a "$GITHUB_STEP_SUMMARY"
-
- # Per-test failure rollup: only when a test job actually failed, download that
- # shard's JUnit so "Check dependency results" can list the failing tests. Skipped
- # on green runs (the common case) to avoid pulling every shard artifact for nothing.
- # continue-on-error so a flaky fetch can't fail this required gate.
- - name: Download shard JUnit results
- if: always() && github.event_name != 'pull_request' && (needs.django.result != 'success' || needs.turbo-tests.result != 'success')
- continue-on-error: true
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- pattern: '{junit-results-backend,product-junit-results}-*'
- path: /tmp/junit-gate/
- # Re-run attempts upload under suffixed names, so superseded attempts must be dropped
- # here or the rollup below lists attempt-1 failures the re-run already recovered.
- - name: Keep only each shard's latest attempt
- if: always() && github.event_name != 'pull_request' && (needs.django.result != 'success' || needs.turbo-tests.result != 'success')
- continue-on-error: true
- run: |
- cd /tmp/junit-gate 2>/dev/null || exit 0
- shopt -s nullglob
- for d in *-attempt[0-9]*; do
- base="${d%-attempt*}"
- latest="$(printf '%s\n' "$base"-attempt[0-9]* | sort -V | tail -1)"
- rm -rf "$base"
- [[ "$d" == "$latest" ]] || rm -rf "$d"
- done
- - name: Check dependency results
- run: |
- # Per-test failure rollup, emitted from THIS step because GitHub only
- # auto-expands the failed step in the log view. Best-effort via `|| true`
- # so a parser hiccup never changes the verdict computed below.
- python3 <<'PY' || true
- import glob, os, xml.etree.ElementTree as ET
-
- ROOT = "/tmp/junit-gate"
- rows = [] # (shard, nodeid, repo_file, message)
-
- def clean(s):
- # Collapse newlines so PR-authored test names can't inject ::workflow-commands::
- return " ".join((s or "").split())
-
- for path in glob.glob(os.path.join(ROOT, "**", "junit-*.xml"), recursive=True):
- artifact = os.path.relpath(path, ROOT).split(os.sep)[0]
- base = os.path.basename(path)
- # Product JUnit is staged as junit-product-<name>.xml; backend keeps its key.
- product = base[len("junit-product-"):-len(".xml")] if base.startswith("junit-product-") else ""
- shard = f"products/{product}" if product else artifact.removeprefix("junit-results-backend-")
- with open(path, "rb") as fh:
- raw = fh.read()
- # Refuse DTD/entity payloads (XXE, billion-laughs). pytest JUnit has neither,
- # so real failures are never dropped; a crafted file is simply skipped.
- if b"<!DOCTYPE" in raw or b"<!ENTITY" in raw:
- continue
- try:
- tree = ET.fromstring(raw)
- except ET.ParseError:
- continue
- for tc in tree.iter("testcase"):
- bad = tc.find("failure")
- if bad is None:
- bad = tc.find("error")
- if bad is None:
- continue
- name = tc.get("name", "")
- file_ = tc.get("file", "") or tc.get("classname", "")
- repo_file = f"products/{product}/{file_}" if product and file_ else file_
- nodeid = clean("::".join(p for p in (repo_file, name) if p))
- msg = clean(bad.get("message") or bad.text or "")[:200]
- rows.append((clean(shard), nodeid, clean(repo_file), msg))
-
- # Print only when something failed; on a green gate this stays silent.
- if rows:
- out = ["### Failing backend & product tests", "", f"{len(rows)} failing test(s):",
- "", "| Shard | Test | Message |", "| --- | --- | --- |"]
- for shard, nodeid, _f, msg in rows[:100]:
- out.append(f"| {shard} | `{nodeid.replace('|', r'\|')}` | {msg.replace('|', r'\|')} |")
- if len(rows) > 100:
- out += ["", f"…and {len(rows) - 100} more."]
- text = "\n".join(out) + "\n"
- summary = os.environ.get("GITHUB_STEP_SUMMARY")
- if summary:
- with open(summary, "a") as fh:
- fh.write(text)
- print(text)
- for shard, nodeid, repo_file, msg in rows[:30]:
- title = f"Backend test failed ({shard})"
- body = f"{nodeid}: {msg}" if msg else nodeid
- loc = f"file={repo_file}," if repo_file else ""
- print(f"::error {loc}title={title}::{body}")
- PY
-
- if [[ "${{ needs.repo-checks.outputs.deterministic_failure }}" == "true" ]]; then
- echo "::error title=Repo checks failed deterministically::Open the repo checks job for the root failure. Backend CI cancelled downstream jobs to save runners; retrying will not fix this."
- fi
-
- if [[ "${{ needs.check-openapi-types.outputs.deterministic_failure }}" == "true" ]]; then
- echo "::error title=OpenAPI type checks failed deterministically::Open the OpenAPI type check job for the root failure. Backend CI cancelled downstream jobs to save runners; retrying will not fix this."
- fi
-
- check_required_result() {
- local label="$1"
- local result="$2"
-
- if [[ "$result" != "success" && "$result" != "skipped" ]]; then
- echo "::error::$label did not succeed (result: '$result')."
- exit 1
- fi
- }
-
- check_required_result "Repo checks" "${{ needs.repo-checks.result }}"
- check_required_result "SDK major guard" "${{ needs.sdk-major-guard.result }}"
- check_required_result "OpenAPI type checks" "${{ needs.check-openapi-types.result }}"
- # Upstream jobs whose failure would otherwise only cascade into django
- # being *skipped*, which this gate reads as success — a green gate with
- # zero backend tests run. Naming them makes the failure hit the gate
- # directly; legitimate skips (frontend-only PRs) and plain successes still pass.
- check_required_result "Change detection" "${{ needs.changes.result }}"
- check_required_result "ClickHouse versions" "${{ needs.get_clickhouse_versions.result }}"
- check_required_result "Django matrix build" "${{ needs.build_django_matrix.result }}"
- check_required_result "Django test matrix" "${{ needs.django.result }}"
- check_required_result "Migration checks" "${{ needs.check-migrations.result }}"
- check_required_result "Turbo discover" "${{ needs.turbo-discover.result }}"
- check_required_result "Product test matrix build" "${{ needs.build-product-test-matrix.result }}"
- check_required_result "Product/Turbo tests" "${{ needs.turbo-tests.result }}"
- check_required_result "Snapshot mode detection" "${{ needs.detect-snapshot-mode.result }}"
- check_required_result "Snapshot commit job" "${{ needs.handle-snapshots.result }}"
-
- echo "All backend and product checks passed."
-
- test-selection-verdict:
- needs: [django, turbo-tests, changes, turbo-discover]
- name: Test selection verdict
- # The verdict scores the selection against the JUnit a run produced, so it needs
- # a test job to have run. changes is skipped on no-ci drafts. Both matrices are
- # skipped on a draft whose selection was untrusted, which defers to the ready run
- # — there is no selection to score and no JUnit to score it against. The decision
- # itself is still recorded, by capture-test-selection.
- if: >-
- !cancelled() && github.event_name == 'pull_request' &&
- needs.changes.result != 'skipped' &&
- (needs.django.result != 'skipped' || needs.turbo-tests.result != 'skipped')
- runs-on: ubuntu-24.04
- timeout-minutes: 10
- permissions:
- contents: read
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- fetch-depth: 1000
- filter: blob:none
-
- - name: Install uv
- uses: ./.github/actions/setup-uv
- with:
- enable-cache: false
-
- - name: Download JUnit artifacts
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- pattern: junit-results-backend-*
- path: /tmp/junit-results/
- continue-on-error: true
-
- # Re-run attempts upload under suffixed names; without this the verdict double-counts
- # re-executed shards and resurrects failures the re-run already recovered.
- - name: Keep only each shard's latest attempt
- continue-on-error: true
- run: |
- cd /tmp/junit-results 2>/dev/null || exit 0
- shopt -s nullglob
- for d in *-attempt[0-9]*; do
- base="${d%-attempt*}"
- latest="$(printf '%s\n' "$base"-attempt[0-9]* | sort -V | tail -1)"
- rm -rf "$base"
- [[ "$d" == "$latest" ]] || rm -rf "$d"
- done
-
- # Reuse the selection artifact from the turbo-discover job rather than
- # re-running the snob script. Falls through if the artifact wasn't
- # produced (the selector did not run or failed) — the next step
- # regenerates it so we still get a shadow verdict for those PRs.
- - name: Download selection artifact
- id: download-selection
- continue-on-error: true
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- name: select-tests-output
- path: /tmp/verdict/
-
- - name: Download the run's sharding plan snapshot
- # The verdict reports selected seconds per segment from .test_durations,
- # which is not in the repo; use the union turbo-discover snapshotted.
- continue-on-error: true
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- name: test-durations-plan
- path: .
-
- - name: Run test selection and verdict
- env:
- DIFF_BASE: ${{ format('{0}^1', github.sha) }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- PR_SHA: ${{ github.event.pull_request.head.sha }}
- run: |
- set -euo pipefail
- mkdir -p /tmp/verdict
-
- if [[ ! -s /tmp/verdict/selection.json ]]; then
- test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
- test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3
- test "$(git rev-parse HEAD^2)" = "$PR_SHA"
- git rev-parse --verify "$DIFF_BASE^{commit}"
-
- uv run tools/snob_backend_test_selection_shadow.py \
- --base-ref "$DIFF_BASE" \
- --pretty \
- > /tmp/verdict/selection.json
- fi
-
- uv run tools/test_selection_verdict.py \
- /tmp/verdict/selection.json \
- /tmp/junit-results/ \
- --summary-path "$GITHUB_STEP_SUMMARY" \
- --pretty \
- > /tmp/verdict/verdict.json
-
- - name: Upload verdict
- if: always()
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- with:
- name: test-selection-verdict-pr${{ github.event.pull_request.number }}
- path: /tmp/verdict/
- retention-days: 90
-
- calculate-running-time:
- name: Calculate running time
- needs: [changes, django_tests]
- runs-on: ubuntu-24.04
- timeout-minutes: 5
- # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks or Dependabot (no secrets access)
- if: >-
- !cancelled() && github.actor != 'dependabot[bot]' && (
- (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') ||
- (github.event_name != 'pull_request' && github.repository == 'PostHog/posthog'))
- && needs.changes.result == 'success'
- permissions:
- contents: read
- actions: read
- steps:
- - name: Get telemetry app token
- id: telemetry-app-token
- if: github.run_attempt == '1'
- continue-on-error: true
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
- with:
- client-id: ${{ vars.GH_APP_TELEMETRY_APP_ID }}
- private-key: ${{ secrets.GH_APP_TELEMETRY_PRIVATE_KEY }}
- - name: Capture running time to PostHog
- if: github.run_attempt == '1'
- continue-on-error: true
- uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0
- with:
- posthog-token: ${{ secrets.POSTHOG_API_TOKEN }}
- event: 'posthog-ci-running-time'
- capture-run-duration: true
- capture-job-durations: true
- github-token: ${{ steps.telemetry-app-token.outputs.token || github.token }}
- status-job: 'Django Tests Pass'
- runner: 'depot'
- - name: Capture running time to DevEx PostHog
- if: github.run_attempt == '1'
- continue-on-error: true
- uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0
- with:
- posthog-token: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }}
- event: 'posthog-ci-running-time'
- capture-run-duration: true
- capture-job-durations: true
- github-token: ${{ steps.telemetry-app-token.outputs.token || github.token }}
- status-job: 'Django Tests Pass'
- runner: 'depot'
-
- capture-test-selection:
- name: Capture test selection
- needs: [turbo-discover]
- runs-on: ubuntu-24.04
- timeout-minutes: 5
- # An empty mode means this run never selected, so there is no decision to report.
- # DevEx project only, mirroring monitor-github-rate-limit; continue-on-error so
- # telemetry never reds CI. Skipped on forks/Dependabot, which never have the secret.
- if: |
- !cancelled()
- && github.actor != 'dependabot[bot]'
- && github.event.pull_request.head.repo.full_name == 'PostHog/posthog'
- && needs.turbo-discover.result == 'success'
- && needs.turbo-discover.outputs.mode != ''
- steps:
- - name: Capture test selection to DevEx PostHog
- # Depot CI evaluates github.run_attempt as an empty string in a job-level if,
- # so this guard stays on the step for the Depot copy of this job to run.
- if: github.run_attempt == '1'
- continue-on-error: true
- uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0
- with:
- posthog-token: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }}
- event: 'posthog-ci-test-selection'
- properties: ${{ needs.turbo-discover.outputs.telemetry }}
-
- report-test-timings:
- name: Report per-test traces
- # django and turbo-tests are here for their results only; django_tests already
- # waits for both, so this adds no wall time.
- needs: [django_tests, django, turbo-tests]
- runs-on: ubuntu-24.04
- continue-on-error: true
- timeout-minutes: 10
- # Runs on every attempt: a re-run passing what an earlier attempt failed is the same-commit
- # flake proof the test-health queue reads, and the shards a re-run re-executed upload under
- # attempt-suffixed artifact names so only they are emitted for the new attempt.
- # Manual master runs and all in-repo PRs. Skip forks (no secrets access)
- # and master pushes (their matrices are skipped, so there are no traces).
- # Both matrices skipped also means no JUnit was uploaded, so there are no
- # traces to emit. This job sits on the tail of the run, so skipping it takes
- # its whole duration off the wall of a draft that ran no tests.
- if: >-
- !cancelled() &&
- github.repository == 'PostHog/posthog' &&
- (needs.django.result != 'skipped' || needs.turbo-tests.result != 'skipped') &&
- (
- (github.event_name != 'pull_request' && github.event_name != 'push' && github.ref == 'refs/heads/master') ||
- (github.event_name == 'pull_request' &&
- github.event.pull_request.head.repo.full_name == 'PostHog/posthog')
- )
- steps:
- # Labelled PRs run the trusted base parser, so the same PR can't swap the script that receives secrets.
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}
- fetch-depth: 1
- - name: Check for trusted timing reporter
- id: timing-reporter
- shell: bash
- run: |
- # The owners resolver is a path dep of the reporter, so a base ref predating it
- # can't resolve its deps. A base ref from before the move to packages/owners-yaml still
- # carries it at tools/owners, so both locations count as available.
- available=false
- if [[ -f .github/scripts/report_test_timings.py && ( -d packages/owners-yaml || -d tools/owners ) ]]; then
- available=true
- fi
- # A re-run may emit only through an attempt-aware reporter: an older script would
- # re-report every attempt-1 shard under the new attempt's resource attributes.
- if [[ "$GITHUB_RUN_ATTEMPT" != "1" ]] && ! grep -q "def partition_run_attempt" .github/scripts/report_test_timings.py 2>/dev/null; then
- available=false
- fi
- if [[ "$available" != "true" ]]; then
- echo "Trusted timing reporter is not available on the checked-out ref for attempt $GITHUB_RUN_ATTEMPT; skipping emit."
- fi
- echo "available=$available" >> "$GITHUB_OUTPUT"
- # Direct install: The trusted PR base used for reporting may predate the local action.
- - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- if: steps.timing-reporter.outputs.available == 'true'
- with:
- version: '0.13.4'
- - name: Download junit artifacts
- if: steps.timing-reporter.outputs.available == 'true'
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- path: ./junit-artifacts
- pattern: '{junit-results-backend,product-junit-results}-*'
- - name: Emit per-test traces
- if: steps.timing-reporter.outputs.available == 'true'
- continue-on-error: true
- env:
- POSTHOG_DEVEX_PROJECT_API_TOKEN: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }}
- # dual emission while per-test traces migrate to the project behind POSTHOG_API_TOKEN
- POSTHOG_CI_TRACES_EXTRA_TOKEN: ${{ secrets.POSTHOG_API_TOKEN }}
- POSTHOG_OTLP_TRACES_ENDPOINT: ${{ vars.POSTHOG_OTLP_TRACES_ENDPOINT || 'https://us.i.posthog.com/i/v1/traces' }}
- run: |
- uv run --script .github/scripts/report_test_timings.py \
- --min-duration-seconds=0.5 \
- ./junit-artifacts
-
- backend-coverage-report:
- name: Backend coverage report
- needs: [changes, turbo-tests, django_tests, turbo-discover]
- runs-on: ubuntu-24.04
- continue-on-error: true # report-only, never block a PR
- timeout-minutes: 10
- # Pinned job-level: this job runs PR-head-controlled scripts with github.token, so its
- # grants must never silently widen if the workflow-level permissions block ever does.
- permissions:
- contents: read
- pull-requests: write # sticky CI report comment
- # In-repo PRs only — fork PRs get a read-only token that can't comment. Runs when either
- # the product (turbo-tests) or core (django_tests) suite ran, so a core-only PR still
- # gets a comment; both skip on drafts, leaving nothing to report.
- # A narrowed run measures only the selected tests, so patch coverage would report
- # gaps that the full suite covers. Report only when the Django matrix ran full.
- # Security invariant: this job runs PR-head-controlled scripts (coverage_report.py,
- # post-coverage-section.mjs) — never add secrets beyond github.token to this job
- # (report-test-timings checks out the base ref for exactly this reason).
- if: >-
- !cancelled() &&
- github.event_name == 'pull_request' &&
- github.event.pull_request.draft == false &&
- needs.turbo-discover.outputs.mode != 'selected' &&
- github.event.pull_request.head.repo.full_name == 'PostHog/posthog' &&
- (needs.turbo-tests.result != 'skipped' || needs.django_tests.result != 'skipped')
- && needs.changes.result == 'success'
- steps:
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- # Default pull_request checkout = the synthetic merge ref — the same tree the
- # product coverage runs measured, so diff line numbers match. The coverage
- # conversion reads its source files from it. Full history for diff-cover;
- # blob:none keeps it light (commit graph only, blobs fetched on demand).
- fetch-depth: 0
- filter: blob:none
- - name: Check for coverage reporter
- id: reporter
- shell: bash
- run: |
- if [[ -f .github/scripts/coverage_report.py ]]; then
- echo "available=true" >> "$GITHUB_OUTPUT"
- else
- echo "available=false" >> "$GITHUB_OUTPUT"
- echo "Coverage reporter not present on this ref; skipping (branch may predate the check)."
- fi
- - name: Download coverage artifacts
- if: steps.reporter.outputs.available == 'true'
- continue-on-error: true # a run where no shard emitted coverage data is not a failure
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- path: ./cov-artifacts
- pattern: coverage-products-*
- - name: Download core coverage artifacts
- if: steps.reporter.outputs.available == 'true'
- continue-on-error: true # core coverage only present once the Core segment ran with --cov
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- with:
- path: ./core-artifacts
- pattern: coverage-core-*
- - uses: ./.github/actions/setup-uv
- with:
- enable-cache: false
- if: steps.reporter.outputs.available == 'true'
- - name: Build coverage report
- # diff-cover is declared inline (PEP 723) in coverage_report.py; uv provisions it.
- if: steps.reporter.outputs.available == 'true'
- run: |
- mkdir -p cov-artifacts core-artifacts
- git fetch --no-tags --filter=blob:none --quiet origin master
- uv run --script .github/scripts/coverage_report.py \
- --artifacts cov-artifacts \
- --core-artifacts core-artifacts \
- --combined-out combined-coverage.xml \
- --compare-branch origin/master \
- --patch-json-out patch-coverage.json \
- --out coverage-comment.md
- - name: Post to CI report comment
- if: steps.reporter.outputs.available == 'true'
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- if [ -f .github/scripts/post-coverage-section.mjs ]; then
- node .github/scripts/post-coverage-section.mjs coverage-comment.md patch-coverage.json
- fi
- - name: Upload patch coverage payload
- if: ${{ always() && steps.reporter.outputs.available == 'true' }}
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
- with:
- name: patch-coverage
- path: |
- patch-coverage.json
- combined-coverage.xml
- coverage-comment.md
- if-no-files-found: ignore
- retention-days: 2
- overwrite: true # artifacts are immutable per run; re-run shards would 409 otherwise
diff --git a/.github/workflows/ci-backend-shadow-drift.yml b/.github/workflows/ci-depot-action-mirrors.yml
similarity index 60%
rename from .github/workflows/ci-backend-shadow-drift.yml
rename to .github/workflows/ci-depot-action-mirrors.yml
index 3cf1a1e0e7cf..30bbbed83c50 100644
--- a/.github/workflows/ci-backend-shadow-drift.yml
+++ b/.github/workflows/ci-depot-action-mirrors.yml
@@ -1,21 +1,12 @@
-name: Backend CI shadow drift check
-
-# Fails when the canonical backend workflow or a mirrored local action changes
-# without a matching update to the Depot shadow. The workflow is hand-maintained
-# (see its header comment for intentional deltas), while the actions are vendored;
-# without this check, canonical drifts silently and the Depot timing trial stops
-# measuring the same work. The check lives in depot_mirrors.py, which
-# `hogli ci:preflight` also runs before a push.
-#
-# Cheap to run: it triggers only on changes to the backend workflow, local actions, Depot files, or the check itself.
-# Always enforced: every pull request's backend tests run on Depot, so the mirror and
-# canonical have to agree on every commit.
+name: Depot action mirrors
+# Depot CI resolves `./.github/actions/<name>` from .depot/actions/<name>, so every local action a
+# Depot workflow uses has a mirror there. This check fails when a canonical action changes without
+# its mirror, or when an exact mirror drifts. `hogli ci:preflight` runs the same check before a push.
on:
pull_request:
paths:
- - .github/workflows/ci-backend-shadow-drift.yml
- - .github/workflows/ci-backend.yml
+ - .github/workflows/ci-depot-action-mirrors.yml
- .github/actions/**
- .depot/**
- tools/hogli-commands/hogli_commands/depot_mirrors.py
@@ -29,7 +20,7 @@ concurrency:
jobs:
check:
- name: Backend CI shadows remain synchronized
+ name: Depot action mirrors remain synchronized
runs-on: ubuntu-24.04
# 5 min absorbs cold-runner startup + the explicit depth=100 fetch of BASE_SHA below.
# `actions/checkout` defaults to depth=1 (HEAD only); a full-history clone is unnecessary
diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml
index bfda50e42c9b..a4451931c67d 100644
--- a/.github/workflows/ci-python.yml
+++ b/.github/workflows/ci-python.yml
@@ -113,7 +113,6 @@ jobs:
# Make sure we run if someone is explicitly change the workflows
- .github/workflows/ci-python.yml
- .github/workflows/ci-dagster.yml
- - .github/workflows/ci-backend.yml
- .github/actions/setup-uv/**
- .depot/actions/setup-uv/**
- '.depot/workflows/*.yml'
diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml
index c212856a8277..8a120f1c85c5 100644
--- a/.github/workflows/ci-scripts.yml
+++ b/.github/workflows/ci-scripts.yml
@@ -21,7 +21,6 @@ on:
- .oxfmtrc*
# turbo-discover.test.js pins the Django segment table to these, and checks
# that every path in the events_json list exists.
- - .github/workflows/ci-backend.yml
- .depot/workflows/ci-backend.yml
- .github/new-events-schema-targets.txt
# The backend test selector and its verdict script gate ci-backend.
diff --git a/posthog/test/repo_invariants/test_events_json_prune_removal.py b/posthog/test/repo_invariants/test_events_json_prune_removal.py
index 17e20ba641a0..ab0643281fd4 100644
--- a/posthog/test/repo_invariants/test_events_json_prune_removal.py
+++ b/posthog/test/repo_invariants/test_events_json_prune_removal.py
@@ -10,7 +10,6 @@ PRUNE_MODULES = (
)
PRUNE_WIRING = (
"conftest.py",
- ".github/workflows/ci-backend.yml",
".github/workflows/ci-backend-update-test-timing.yml",
".depot/workflows/ci-backend.yml",
)
diff --git a/posthog/test/repo_invariants/test_signals_scout_tool_citations.py b/posthog/test/repo_invariants/test_signals_scout_tool_citations.py
index cf6ec2c7e357..a4c1f9c067e3 100644
--- a/posthog/test/repo_invariants/test_signals_scout_tool_citations.py
+++ b/posthog/test/repo_invariants/test_signals_scout_tool_citations.py
@@ -195,8 +195,8 @@ def citation_inputs() -> list[PurePosixPath]:
return [PurePosixPath(path.resolve().relative_to(root).as_posix()) for path in paths]
[email protected]("workflow", [".github/workflows/ci-backend.yml", ".depot/workflows/ci-backend.yml"])
-def test_repo_checks_runs_when_a_citation_input_changes(workflow: str) -> None:
+def test_repo_checks_runs_when_a_citation_input_changes() -> None:
+ workflow = ".depot/workflows/ci-backend.yml"
globs = repo_invariants_filter(workflow)
# full_match keeps `*` inside one path segment, as picomatch in the paths-filter action does.
# fnmatch lets `*` cross `/`, so it would pass a glob that the action never fires on.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.py
index 4d461d350cd7..8c8a0c0ef0e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_ci_core_coupled_sources.py
@@ -44,7 +44,7 @@ _GENERATED_CONFIG_REFERENCE = re.compile(r"(?:[\w.]+\.)?sources\.generated_confi
def _repo_root() -> Path:
for parent in Path(__file__).resolve().parents:
- if (parent / ".github" / "workflows" / "ci-backend.yml").exists():
+ if (parent / ".depot" / "workflows" / "ci-backend.yml").exists():
return parent
raise RuntimeError("repo root not found")
diff --git a/tools/hogli-commands/hogli_commands/ci_preflight.py b/tools/hogli-commands/hogli_commands/ci_preflight.py
index 6ec3ef0ddcd2..6bef9ba5f080 100644
--- a/tools/hogli-commands/hogli_commands/ci_preflight.py
+++ b/tools/hogli-commands/hogli_commands/ci_preflight.py
@@ -378,9 +378,9 @@ DIFF_CHECKS: list[DiffCheck] = [
]
-# A change under these paths can fail .github/workflows/ci-backend-shadow-drift.yml, which runs
+# A change under these paths can fail .github/workflows/ci-depot-action-mirrors.yml, which runs
# the same depot_mirrors check. Running it here lands the failure pre-push instead of in CI.
-SHADOW_DRIFT_TRIGGERS = [".github/workflows/ci-backend.yml", ".github/actions/**", ".depot/**"]
+DEPOT_MIRROR_TRIGGERS = [".github/actions/**", ".depot/**"]
def _has_node_modules() -> bool:
@@ -864,7 +864,7 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,
# A check that can only warn is not worth its run time in the pre-push hook.
if chk.matched and not (strict and chk.soft and chk.run is not None):
triggered.append(chk)
- shadow_drift_triggered = any(matches_globs(path, SHADOW_DRIFT_TRIGGERS) for path in files)
+ depot_mirrors_triggered = any(matches_globs(path, DEPOT_MIRROR_TRIGGERS) for path in files)
results: list[dict[str, Any]] = []
failures = 0
@@ -888,15 +888,15 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,
click.secho(f" {_ICON[depth_status]} [stack-depth] open layers in this gh stack", fg=_COLOR[depth_status])
click.echo(f" {depth_detail}")
- if shadow_drift_triggered:
+ if depot_mirrors_triggered:
violations = mirror_violations(REPO_ROOT, set(files))
drift_status: Status = "fail" if violations else "pass"
drift_detail = " · ".join(violations[:3]) if violations else "depot mirrors in sync"
failures += drift_status == "fail"
- results.append({"check": "shadow-drift", "status": drift_status, "files": 1, "detail": drift_detail})
+ results.append({"check": "depot-mirrors", "status": drift_status, "files": 1, "detail": drift_detail})
if not as_json:
click.secho(
- f" {_ICON[drift_status]} [shadow-drift] depot shadow drift (.depot mirrors of backend CI)",
+ f" {_ICON[drift_status]} [depot-mirrors] depot action mirrors (.depot/actions)",
fg=_COLOR[drift_status],
)
click.echo(f" {drift_detail}")
@@ -914,7 +914,7 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,
summary = {
"changed_files": len(files),
- "triggered": (["shadow-drift"] if shadow_drift_triggered else []) + [c.key for c in triggered],
+ "triggered": (["depot-mirrors"] if depot_mirrors_triggered else []) + [c.key for c in triggered],
"failures": failures,
"advisories": advisories,
"mode": "fix" if do_fix else ("strict" if strict else "advisory"),
@@ -924,7 +924,7 @@ def ci_preflight(do_fix: bool, strict: bool, against: str | None, as_json: bool,
if as_json:
click.echo(json.dumps(summary))
else:
- if not triggered and not shadow_drift_triggered:
+ if not triggered and not depot_mirrors_triggered:
click.secho(" ✓ Nothing in this diff maps to a known CI failure class.", fg="green")
click.echo()
click.echo(
diff --git a/tools/hogli-commands/hogli_commands/depot_mirrors.py b/tools/hogli-commands/hogli_commands/depot_mirrors.py
index 6ebc1b34a639..bc3200ebc6c2 100644
--- a/tools/hogli-commands/hogli_commands/depot_mirrors.py
+++ b/tools/hogli-commands/hogli_commands/depot_mirrors.py
@@ -1,6 +1,6 @@
-"""Keeps the Depot shadow of backend CI in sync with its canonical workflow and local actions.
+"""Keeps the Depot CI mirrors of the local actions in sync with their canonical copies.
-`.github/workflows/ci-backend-shadow-drift.yml` runs this file with the runner's python3 before any
+`.github/workflows/ci-depot-action-mirrors.yml` runs this file with the runner's python3 before any
dependency install, and `hogli ci:preflight` imports it, so it must stay standard library only.
"""
@@ -11,8 +11,6 @@ import sys
import subprocess
from pathlib import Path
-CANONICAL_WORKFLOW = ".github/workflows/ci-backend.yml"
-SHADOW_WORKFLOW = ".depot/workflows/ci-backend.yml"
# These mirrors carry intentional deltas that each mirror's header documents.
# Every other mirror stays byte-identical to its canonical action.
DELTA_MIRRORS = frozenset({"commit-snapshots", "pnpm-install", "setup-pnpm"})
@@ -30,11 +28,6 @@ def _file_bytes(directory: Path) -> dict[Path, bytes]:
def mirror_violations(root: Path, changed: set[str]) -> list[str]:
violations: list[str] = []
- if CANONICAL_WORKFLOW in changed and SHADOW_WORKFLOW not in changed:
- violations.append(
- f"{CANONICAL_WORKFLOW} changed without a matching update to {SHADOW_WORKFLOW}. "
- "Mirror the change, or document it as an intentional delta in the shadow's header."
- )
for name in sorted(used_mirrors(root)):
canonical = f".github/actions/{name}/"
mirror = f".depot/actions/{name}/"
@@ -54,14 +47,10 @@ def mirror_violations(root: Path, changed: set[str]) -> list[str]:
def main(base_sha: str) -> int:
diff = subprocess.run(["git", "diff", "--name-only", base_sha, "HEAD"], check=True, capture_output=True, text=True)
changed = set(diff.stdout.splitlines())
- if SHADOW_WORKFLOW in changed and CANONICAL_WORKFLOW not in changed:
- sys.stdout.write(
- f"::notice::Only {SHADOW_WORKFLOW} changed. That is allowed, for example to tune a Depot-only knob.\n"
- )
violations = mirror_violations(Path.cwd(), changed)
sys.stdout.writelines(f"::error::{violation}\n" for violation in violations)
if not violations:
- sys.stdout.write("The Depot shadow is in sync.\n")
+ sys.stdout.write("The Depot action mirrors are in sync.\n")
return 1 if violations else 0
diff --git a/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.py b/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.py
index 7c19b43d3b9e..e845a84d1051 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_ci_preflight.py
@@ -294,7 +294,7 @@ class TestStalenessRisks:
["migrations added on both sides in posthog/migrations"],
),
(["posthog/api/ours.py"], ["posthog/api/theirs.py"], [], ["master also changed build:openapi"]),
- (["a.py"], [".github/workflows/ci-backend.yml"], [], ["CI workflows changed on master (1 file(s))"]),
+ (["a.py"], [".github/workflows/ci-python.yml"], [], ["CI workflows changed on master (1 file(s))"]),
],
)
def test_risk_derivation(
@@ -396,7 +396,7 @@ class TestWorkspaceScopedLockfile:
assert "products/desktop: needs node" in detail
-class TestShadowDriftCheck:
+class TestDepotMirrorsCheck:
@pytest.mark.parametrize(
"changed,expected_exit,expected_fragment",
[
@@ -427,4 +427,4 @@ class TestShadowDriftCheck:
if expected_fragment:
assert expected_fragment in result.output
else:
- assert "shadow-drift" not in result.output
+ assert "depot-mirrors" not in result.output
diff --git a/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.py b/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.py
index 984e196b0403..a6372e66ff45 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_depot_mirrors.py
@@ -58,12 +58,6 @@ class TestMirrorViolations:
"but .depot/actions/pnpm-install/ does not exist",
id="used-mirror-deleted",
),
- pytest.param(
- {".github/workflows/ci-backend.yml"},
- None,
- "without a matching update to .depot/workflows/ci-backend.yml",
- id="canonical-workflow-only",
- ),
],
)
def test_reports_drift(
diff --git a/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.py b/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.py
index 71e2d0966d2e..051dbab4e77c 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_preflight_checks.py
@@ -316,7 +316,7 @@ class TestSemgrepDevex:
assert mock_popen.call_args.kwargs["start_new_session"] is True
-WORKFLOW = ".github/workflows/ci-backend.yml"
+WORKFLOW = ".github/workflows/ci-python.yml"
PRODUCT_FILE = "products/x/backend/api.py"
EVERY_LANE = ["fe:core", "py:core", "rust:core"]
diff --git a/tools/snob_backend_test_selection_shadow.py b/tools/snob_backend_test_selection_shadow.py
index 4da6fd102a4f..884b64464d85 100755
--- a/tools/snob_backend_test_selection_shadow.py
+++ b/tools/snob_backend_test_selection_shadow.py
@@ -80,7 +80,7 @@ FULL_RUN_PATTERNS = (
# see that edge. turbo-discover.js documents this as a live invariant.
".test_quarantine.json",
# CI / Docker infrastructure
- ".github/workflows/ci-backend.yml",
+ ".depot/workflows/ci-backend.yml",
# A new path in the events_json list has to run on the PR that adds it.
".github/new-events-schema-targets.txt",
".github/clickhouse-versions.json",
diff --git a/tools/workflow-plan/tests/backend-diff.test.ts b/tools/workflow-plan/tests/backend-diff.test.ts
index 749c2ead3e73..75ab3cfd8940 100644
--- a/tools/workflow-plan/tests/backend-diff.test.ts
+++ b/tools/workflow-plan/tests/backend-diff.test.ts
@@ -8,8 +8,7 @@ import { type Context, type JsonValue, evaluateCondition, evaluateTemplate, plan
import { type RawStep, type Workflow, loadWorkflow, planWorkflow } from '../src/plan.ts'
import { REPO_ROOT, allFiltersChanged, mergeQueue, pullRequest } from '../src/scenarios.ts'
-const WORKFLOWS = ['.github/workflows/ci-backend.yml', '.depot/workflows/ci-backend.yml']
-const [GITHUB_BACKEND, DEPOT_BACKEND] = WORKFLOWS
+const DEPOT_BACKEND = '.depot/workflows/ci-backend.yml'
const functions = planFunctions({ dependenciesSucceeded: true, dependenciesFailed: false, cancelled: false })
const lowerFile = 'products/engineering_analytics/backend/lower.py'
const layerFiles = [
@@ -135,10 +134,10 @@ function requiredGate(wf: Workflow, cwd: string, context: Context): SpawnSyncRet
}
describe('Backend CI comparison boundaries', () => {
- it.each(WORKFLOWS)('%s selects a stack layer without counting newer trunk files', (file) => {
+ it('selects a stack layer without counting newer trunk files', () => {
const repo = createGraph()
try {
- const wf = loadWorkflow(path.join(REPO_ROOT, file))
+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))
const context = prContext(repo.merge, repo.head, 'lower')
const discovery = stepEnv(wf, step(wf, 'Discover products to test'), context)
const selected = repo.git(
@@ -182,10 +181,10 @@ describe('Backend CI comparison boundaries', () => {
}
})
- it.each(WORKFLOWS)('%s uses the pinned queue base and disables the PR selector', (file) => {
+ it('uses the pinned queue base and disables the PR selector', () => {
const repo = createGraph()
try {
- const wf = loadWorkflow(path.join(REPO_ROOT, file))
+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))
const context = prContext(repo.queueMerge, repo.merge, 'master', true)
const discovery = stepEnv(wf, step(wf, 'Discover products to test'), context)
expect(discovery.TURBO_SCM_BASE).toBe(`${repo.queueMerge}^1`)
@@ -221,7 +220,7 @@ describe('Backend CI comparison boundaries', () => {
(failure) => {
const repo = createGraph()
try {
- const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND!))
+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))
const context = prContext(repo.merge, failure === 'wrong-head' ? repo.lower : repo.head, 'lower')
if (failure === 'wrong-checkout') {
repo.git('checkout', '--detach', repo.head)
@@ -269,7 +268,7 @@ describe('Backend CI comparison boundaries', () => {
])('regenerates a missing verdict selection for $name', ({ queued }) => {
const repo = createGraph()
try {
- const wf = loadWorkflow(path.join(REPO_ROOT, GITHUB_BACKEND!))
+ const wf = loadWorkflow(path.join(REPO_ROOT, DEPOT_BACKEND))
const sha = queued ? repo.queueMerge : repo.merge
const context = prContext(sha, queued ? repo.merge : repo.head, queued ? 'master' : 'lower', queued)
const target = wf.jobs['test-selection-verdict']!.steps!.find(
diff --git a/tools/workflow-plan/tests/workflows.test.ts b/tools/workflow-plan/tests/workflows.test.ts
index d36671ec72f8..afd33d824ff2 100644
--- a/tools/workflow-plan/tests/workflows.test.ts
+++ b/tools/workflow-plan/tests/workflows.test.ts
@@ -88,11 +88,10 @@ const suite = (file: string, selectors: Stubs = {}): ExpectationBuilder => {
})
}
const DEPOT_BACKEND = '.depot/workflows/ci-backend.yml'
-const backend = suite('ci-backend.yml', backendSelectors)
const depotBackend = suite(DEPOT_BACKEND, backendSelectors)
const frontend = suite('ci-frontend.yml', frontendSelectors)
const deltalite = suite('build-deltalite.yml')
-const PINNED_WORKFLOWS = ['ci-backend.yml', DEPOT_BACKEND, 'ci-frontend.yml']
+const PINNED_WORKFLOWS = [DEPOT_BACKEND, 'ci-frontend.yml']
interface ReleaseWorkflow {
file: string
@@ -163,25 +162,6 @@ const allJobsExcept = (file: string, kept: string[]): string[] =>
Object.keys(workflow(file).jobs).filter((id) => !kept.includes(id))
const EXPECTATIONS: Expectation[] = [
- // Depot CI runs every pull request event. GitHub Actions keeps only the `Django Tests Pass`
- // job, which points at the Depot gate, so every other job stands down on a pull request.
- ...(
- [
- ['draft PR', pullRequest({ draft: true }), ['dynamic-ci-filter']],
- ['ready PR', pullRequest(), ['dynamic-ci-filter']],
- ['merge queue', mergeQueue(), []],
- ['fork PR', pullRequest({ fork: true }), []],
- ['draft PR labeled no-ci', pullRequest({ draft: true, labels: ['no-ci'] }), []],
- ] as const
- ).map(([name, github, alsoRuns]) =>
- backend(
- { name, github },
- {
- runs: ['django_tests', ...alsoRuns],
- skipped: allJobsExcept('ci-backend.yml', ['django_tests', ...alsoRuns]),
- }
- )
- ),
// The master push runs on Depot CI: the per-commit checks and the schema dump, no matrices.
depotBackend(
{ name: 'master push', github: push() },
@@ -208,11 +188,9 @@ const EXPECTATIONS: Expectation[] = [
],
}
),
- ...[backend, depotBackend].map((build) =>
- build(
- { name: 'ready PR, superseded and cancelled', cancelled: true },
- { results: { django_tests: 'cancelled' } }
- )
+ depotBackend(
+ { name: 'ready PR, superseded and cancelled', cancelled: true },
+ { results: { django_tests: 'cancelled' } }
),
depotBackend(
{ name: 'draft PR', github: pullRequest({ draft: true }) },
@@ -526,10 +504,22 @@ const STEP_EXPECTATIONS: StepExpectation[] = [
scenario: { name: 'hourly schedule', github: schedule() },
runs: false,
},
- // The GitHub Actions backend workflow skips change detection on pull requests, which run on Depot CI.
- ...PINNED_WORKFLOWS.filter((file) => file !== 'ci-backend.yml').flatMap((file) => [
- { file, job: 'changes', step: 'filter', scenario: { name: 'ready PR', github: pullRequest() }, runs: true },
- { file, job: 'changes', step: 'app-token', scenario: { name: 'ready PR', github: pullRequest() }, runs: true },
+ ...PINNED_WORKFLOWS.flatMap((file) => [
+ {
+ file,
+ job: 'changes',
+ step: 'filter',
+ scenario: { name: 'ready PR', github: pullRequest() },
+ runs: true,
+ },
+ { file, job: 'changes', step: 'filter', scenario: { name: 'master push', github: push() }, runs: false },
+ {
+ file,
+ job: 'changes',
+ step: 'app-token',
+ scenario: { name: 'ready PR', github: pullRequest() },
+ runs: true,
+ },
{
file,
job: 'changes',
@@ -538,13 +528,6 @@ const STEP_EXPECTATIONS: StepExpectation[] = [
runs: false,
},
]),
- ...PINNED_WORKFLOWS.map((file) => ({
- file,
- job: 'changes',
- step: 'filter',
- scenario: { name: 'master push', github: push() },
- runs: false,
- })),
{ file: 'ci-e2e-playwright.yml', job: 'changes', step: 'schema-key', scenario: E2E_DISPATCH, runs: true },
{ file: 'ci-e2e-playwright.yml', job: 'playwright', step: 'schema-cache', scenario: E2E_DISPATCH, runs: true },
]
@@ -659,7 +642,7 @@ describe('CI workflow run plans', () => {
expect(testStep?.run).toMatch(/\bgo test\s+-count=1\b/)
})
- it('Backend CI runs once every hour, on Depot CI only', () => {
+ it('Backend CI runs once every hour', () => {
const crons = (file: string): string[] =>
((loadWorkflow(path.join(REPO_ROOT, file)).on as { schedule?: { cron: string }[] }).schedule ?? []).map(
(entry) => entry.cron
@@ -669,7 +652,6 @@ describe('CI workflow run plans', () => {
'23 */3 * * *',
'23 1,2,4,5,7,8,10,11,13,14,16,17,19,20,22,23 * * *',
])
- expect(crons('.github/workflows/ci-backend.yml')).toEqual([])
})
it.each(['refs/heads/master', ''])('the hourly Depot run keeps master coverage with ref %j', (ref) => {