:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Resolve a team's GitHub warehouse tables at query time.
A warehouse table's real name is ``ExternalDataSource.prefix + <synced endpoint table>``,
and ``prefix`` is free text the user sets when connecting the source. The product can
therefore never assume a fixed ``github_*`` table name — a team that connected GitHub
with prefix ``devex_eng_analytics`` lands its data in ``devex_eng_analyticsgithub_pull_requests``.
This resolver walks the team's connected GitHub source(s) to their ``pull_requests`` /
``workflow_runs`` schemas and returns the actual ``DataWarehouseTable`` names the curated
builders should read. Names are resolved exactly once per request (in the logic layer)
and threaded down into the builders, so a request hits the warehouse models a single time.
A team reaches a repo in one of two ways, handled uniformly here: one source per repository
(each with its own bare endpoint names), or one source syncing several repositories (the
multi-repo GitHub source, whose added repos carry repo-qualified ``owner/repo.endpoint`` schema
names while its original repo keeps bare names). Every schema row is grouped under its repo via
the warehouse_sources facade, so both shapes resolve the same way. A caller may pass ``source_id``
to read a specific source; otherwise the oldest source's repos are tried first, or — when a
``repo`` ('owner/name') is passed — the matching repo across all sources is preferred.
"""
import re
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, replace
from datetime import datetime
from typing import TYPE_CHECKING, NamedTuple
from uuid import UUID
from django.db.models import QuerySet
from posthog.dataclasses import frozen
from posthog.models.team import Team
from products.engineering_analytics.backend.facade.contracts import (
CIDataFreshness,
GitHubSource,
GitHubSourceNotConnectedError,
)
from products.engineering_analytics.backend.logic.views import depot_ci, trunk_quarantined_tests, workflow_jobs
from products.engineering_analytics.backend.logic.views.source_schema import OPTIONAL_WORKFLOW_RUNS_COLUMNS
from products.warehouse_sources.backend.facade.models import ExternalDataSchema, ExternalDataSource
from products.warehouse_sources.backend.facade.source_inputs import SourceInputsStore, UnreadableSourceInputsError
from products.warehouse_sources.backend.facade.sources import github_schema_repo_endpoint
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
if TYPE_CHECKING:
from products.access_control.backend.facade.user_access_control import UserAccessControl
# GitHub source endpoints (``ExternalDataSchema.name``) backing the curated builders. The
# materialized table for each is ``prefix + "github_" + endpoint``, e.g. with prefix
# ``devex_eng_analytics`` the pull-requests table is ``devex_eng_analyticsgithub_pull_requests``.
PULL_REQUESTS_SCHEMA = "pull_requests"
WORKFLOW_RUNS_SCHEMA = "workflow_runs"
# Job-level CI (queue time, per-job duration, runner tier). Optional — the source/sync lands
# separately, so reads must degrade gracefully (no jobs) rather than require it like the pair above.
WORKFLOW_JOBS_SCHEMA = "workflow_jobs"
# GitHub org team membership (login → team slug), the author→team key behind team-level merge
# timing. Optional and off by default at the source (needs the org Members:Read grant), so reads
# must degrade gracefully (no membership data) exactly like workflow_jobs.
TEAM_MEMBERS_SCHEMA = "team_members"
# Immutable issue/PR events, the substrate for ready-to-merge timing. Optional at the source,
# so reads must degrade gracefully (no transition data) exactly like workflow_jobs.
ISSUE_EVENTS_SCHEMA = "issue_events"
# Deploy requests and their append-oriented status history, the DORA substrate. Both are
# optional at the source and only useful together (a deployment without statuses never
# succeeded or failed), so reads must degrade gracefully when either is unsynced.
DEPLOYMENTS_SCHEMA = "deployments"
DEPLOYMENT_STATUSES_SCHEMA = "deployment_statuses"
# Submitted pull request reviews, the substrate for the approval split on the author page. Optional
# at the source, so reads must degrade gracefully (no review data) exactly like issue_events.
REVIEWS_SCHEMA = "reviews"
# The Depot source's one table: a row per Depot CI job attempt. Optional, and resolved from a Depot
# source rather than the GitHub one, so reads degrade to GitHub-only CI exactly like workflow_jobs.
DEPOT_JOB_ATTEMPTS_SCHEMA = "job_attempts"
# GitHub adds this column to an issue event only for a team review request, so it lands only once some
# pull request in the repo requested a team, and a read of it fails before that.
REQUESTED_TEAM_COLUMN = "requested_team"
# GitHub puts the membership role (``maintainer`` / ``member``) on a team member row, but it is not in
# the endpoint's documented user object, so a snapshot can land without it. Probed per table like
# REQUESTED_TEAM_COLUMN, and a read without it reports every member as a plain member.
MEMBER_ROLE_COLUMN = "role"
# The curated endpoints we resolve per repo. A source's other synced endpoints (issues, commits,
# teams, …) are irrelevant to the CI/PR read layer and dropped during grouping.
_CURATED_ENDPOINTS = frozenset(
{
PULL_REQUESTS_SCHEMA,
WORKFLOW_RUNS_SCHEMA,
WORKFLOW_JOBS_SCHEMA,
TEAM_MEMBERS_SCHEMA,
ISSUE_EVENTS_SCHEMA,
DEPLOYMENTS_SCHEMA,
DEPLOYMENT_STATUSES_SCHEMA,
REVIEWS_SCHEMA,
}
)
# Resolved names are interpolated into HogQL ``FROM`` clauses. Warehouse table names are
# always plain identifiers (the prefix is validated to ``[A-Za-z0-9_]`` at connect time and
# the rest is the fixed ``github_<endpoint>`` suffix), so reject anything else defensively
# rather than trust an unexpected name into SQL.
_IDENTIFIER = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*\Z")
@dataclass(frozen=True)
class GitHubTables:
"""The selected GitHub source identity and warehouse tables the curated layer reads."""
pull_requests: str
workflow_runs: str
# Optional: present only once the job-level source is synced; None means "no jobs data".
workflow_jobs: str | None = None
# Optional: present only once org team membership is synced; None means "no membership data".
team_members: str | None = None
# Optional: present only once issue events are synced; None means "no transition data".
issue_events: str | None = None
# Optional pair: present only once deploys are synced; None means "no deploy data". Only
# useful together, so consumers gate on both.
deployments: str | None = None
deployment_statuses: str | None = None
# Optional: present only once reviews are synced; None means "no review data".
reviews: str | None = None
# True when the issue-events table has the requested_team column (see REQUESTED_TEAM_COLUMN).
issue_events_team_requests: bool = False
# The OPTIONAL_WORKFLOW_RUNS_COLUMNS that the runs table has. A read selects NULL for the others.
workflow_runs_optional_columns: frozenset[str] = frozenset()
# When the last completed sync of each CI table started. None when the table never synced.
workflow_runs_synced_at: datetime | None = None
workflow_jobs_synced_at: datetime | None = None
# Used to scope cross-store reads such as CI traces to the selected source's repository.
repository: str = ""
# The source these tables came from, already filtered by the caller's per-source warehouse RBAC.
# A read outside the warehouse that needs the repository's credential takes it from this source
# alone, so it can never reach a credential of a source the caller may not use.
source_id: str = ""
def resolve_github_tables(
*,
team: Team,
source_id: str | None = None,
repo: str | None = None,
user_access_control: "UserAccessControl | None" = None,
) -> GitHubTables:
"""Resolve the team's curated GitHub table names from its warehouse models.
With ``source_id``, reads that specific connected GitHub source; otherwise picks the oldest
source's first usable repo with both endpoints synced (oldest = the team's first/established
connection) — deterministic when a team has more than one source or a source syncs more than
one repo. Without ``source_id``, a repo that ``list_github_sources`` reports as synced wins over
one it does not, so the default matches the first synced entry of the list; the oldest usable
repo is the fallback when no listed repo is synced. Raises ``GitHubSourceNotConnectedError``
when no matching usable repo exists (the presentation layer maps it to a 400, so the UI prompts
to connect a source and an agent gets an actionable error), or ``ValueError`` when ``source_id``
is not a UUID.
``repo`` ('owner/name') scopes a repo-specific call — a team with several repos otherwise always
resolves the oldest, so a repo-scoped read (e.g. ``resolve_branch``, or the picker selecting a
specific repo of a multi-repo source) would search the wrong repo's tables. Repos whose resolved
identity equals it (case-insensitively) are tried first — including within a chosen ``source_id``,
so a ``(source_id, repo)`` pair reads that exact repo; the rest still follow as a fallback, since
a bare row's repo can be empty while its data holds the repo.
``user_access_control`` enforces the requesting user's per-source warehouse RBAC (applied in
``_github_sources``): a denied ``source_id`` raises (400) and the default-oldest path skips it.
The curated HogQL runs team-scoped with no user and HogQL does not enforce per-user ACL on
warehouse tables, so honoring it here is the only way the read path can. ``None`` (system/
Temporal/CLI contexts with no request user) skips filtering — team scoping still holds.
"""
queryset = _github_sources(team, user_access_control)
if source_id is not None:
queryset = queryset.filter(id=_as_source_uuid(source_id))
candidates: Iterable[_RepoCandidate] = _repo_candidates(team=team, sources=queryset)
if source_id is None:
# Read what the sources list calls synced first. An old source can still hold tables for a repo
# it no longer lists, and taking it by age alone reads stale data instead of the live source.
candidates = _listed_first(candidates)
if repo:
# An explicit repo must never silently resolve a *different* named repo: the picker lists a
# source's repos before they finish syncing, so a not-yet-complete pick should surface the
# not-connected 400 rather than fall through to a sibling repo (mixing two repos' metrics).
# Keep exact matches; only when there is none, fall back to bare/unattributed ('' repo)
# rows — a legacy single-repo source's rows carry no repo, so a branch-hint read still
# reaches them. Once an exact match exists, bare rows are off the table too: a half-synced
# exact match must surface not-connected, not another source's unattributed data.
wanted = repo.casefold()
materialized = list(candidates)
exact = [c for c in materialized if c.repository.casefold() == wanted]
candidates = exact if exact else [c for c in materialized if c.repository == ""]
for candidate in candidates:
tables = candidate.tables.names
pull_requests = tables.get(PULL_REQUESTS_SCHEMA)
workflow_runs = tables.get(WORKFLOW_RUNS_SCHEMA)
# Both endpoints are required together, by design: every read surface (cards, PR list,
# lifecycle) joins workflow_runs for CI status and the push / re-run rollup, so a repo
# with only pull_requests synced can't serve a complete result. The trade-off is that a
# half-synced repo makes the whole product 400 (e.g. while workflow_runs is still
# backfilling) instead of degrading to PRs-without-CI. Relaxing this to a graceful
# PR-only mode (null CI columns) is a deliberate future change, not handled here.
if pull_requests and workflow_runs:
# workflow_jobs / team_members are optional: included when synced, None otherwise
# (jobs degrade to empty, membership-keyed reads degrade to "no membership data").
return GitHubTables(
pull_requests=pull_requests,
workflow_runs=workflow_runs,
workflow_jobs=tables.get(WORKFLOW_JOBS_SCHEMA),
team_members=tables.get(TEAM_MEMBERS_SCHEMA),
issue_events=tables.get(ISSUE_EVENTS_SCHEMA),
deployments=tables.get(DEPLOYMENTS_SCHEMA),
deployment_statuses=tables.get(DEPLOYMENT_STATUSES_SCHEMA),
reviews=tables.get(REVIEWS_SCHEMA),
issue_events_team_requests=candidate.tables.issue_events_team_requests,
workflow_runs_optional_columns=candidate.tables.workflow_runs_optional_columns,
workflow_runs_synced_at=candidate.tables.synced_at.get(WORKFLOW_RUNS_SCHEMA),
workflow_jobs_synced_at=candidate.tables.synced_at.get(WORKFLOW_JOBS_SCHEMA),
repository=candidate.repository,
source_id=candidate.source_id,
)
if source_id is not None:
raise GitHubSourceNotConnectedError(_NO_SELECTED_SOURCE)
raise GitHubSourceNotConnectedError()
@dataclass(frozen=True)
class JobSourceTables:
"""One qualifying repo's job-level warehouse tables, for the exposed per-job views. Read runs and
jobs through ``runs_source`` and ``jobs_source``, which add the repo's Depot CI; the ``github_``
tables hold GitHub Actions alone."""
github_workflow_jobs: str
github_workflow_runs: str
# Optional: these views qualify on jobs + runs alone, so a repo can reach them with no PR
# snapshot. Consumers that enrich from it (default-branch PR attribution) degrade without it.
pull_requests: str | None = None
issue_events: str | None = None
reviews: str | None = None
source_id: str = ""
repository: str = ""
depot_job_attempts: depot_ci.DepotJobAttempts | None = None
workflow_runs_optional_columns: frozenset[str] = frozenset()
@property
def runs_source(self) -> str:
"""The runs to read: the GitHub runs table plus this repo's Depot CI runs when synced."""
return depot_ci.with_depot_runs(
self.github_workflow_runs,
self.depot_job_attempts,
self.pull_requests,
self.github_workflow_jobs,
optional_columns=self.workflow_runs_optional_columns,
)
@property
def jobs_source(self) -> workflow_jobs.JobsTable:
"""The jobs to read: the GitHub jobs table plus this repo's Depot CI job attempts when synced."""
return depot_ci.with_depot_jobs(self.github_workflow_jobs, self.depot_job_attempts, self.github_workflow_runs)
def resolve_job_source_tables(team: Team) -> list[JobSourceTables]:
"""Job-level tables for every synced repo with BOTH the jobs and runs endpoints.
Used to build the exposed per-job views (cost, CI job history, CI failures), which union across
all of a team's qualifying repos. Unlike ``resolve_github_tables`` (which needs pull_requests +
workflow_runs and returns one repo), this needs workflow_jobs + workflow_runs and returns all of
them, because those views shouldn't collapse a team's repos to one over a PR dependency they
only partly have. A multi-repo source contributes one entry per repo, so the views stay complete when
one source syncs several repos. Userless (the view sync runs in a system/Temporal context);
team scoping is the boundary.
"""
entries: list[tuple[str, JobSourceTables]] = []
for source in _github_sources(team):
for repository, repo_tables in _synced_tables_by_repo(team=team, source=source).items():
tables = repo_tables.names
runs = tables.get(WORKFLOW_RUNS_SCHEMA)
jobs = tables.get(WORKFLOW_JOBS_SCHEMA)
if runs and jobs:
entry = JobSourceTables(
github_workflow_jobs=jobs,
github_workflow_runs=runs,
pull_requests=tables.get(PULL_REQUESTS_SCHEMA),
issue_events=tables.get(ISSUE_EVENTS_SCHEMA),
reviews=tables.get(REVIEWS_SCHEMA),
source_id=str(source.id),
repository=repository,
workflow_runs_optional_columns=repo_tables.workflow_runs_optional_columns,
)
entries.append((repository, entry))
# The views union every entry, so a repository that two GitHub sources sync takes its Depot table on
# one entry only, or every Depot job and its cost would count twice. Entries with the PR snapshot go
# first, because the friction view reads only those entries.
entries.sort(key=lambda repository_entry: repository_entry[1].pull_requests is None)
depot_tables = resolve_depot_job_attempts_tables(team)
return [replace(entry, depot_job_attempts=depot_tables.pop(repository, None)) for repository, entry in entries]
@dataclass(frozen=True)
class TeamMembershipTable:
"""The synced org team-membership snapshot: its warehouse table and whether it carries roles."""
table: str
has_role: bool
def resolve_team_membership_table(
*, team: Team, user_access_control: "UserAccessControl | None" = None
) -> TeamMembershipTable | None:
"""The team's synced ``team_members`` table, or None when no connected source syncs it.
Deliberately narrower than ``resolve_github_tables``: turning a team slug into logins needs
neither ``pull_requests`` nor ``workflow_runs``, so requiring them would make routing fail on a
source that syncs the roster alone, or while the PR endpoints are still backfilling. Membership is
org-scoped, so every repo of every source answers with the same roster and the first match wins.
The endpoint is off by default (it needs the org Members grant), so None is the normal state and
callers degrade to "membership isn't synced" rather than "that team has nobody on it".
"""
for source in _github_sources(team, user_access_control):
legacy_repo = _source_repository(source) or None
for schema in _synced_schemas(team=team, source=source):
_, endpoint = github_schema_repo_endpoint(schema.schema_metadata, schema.name, legacy_repo)
if endpoint != TEAM_MEMBERS_SCHEMA:
continue
table = schema.table
if table is None or table.deleted or not _IDENTIFIER.match(table.name):
continue
return TeamMembershipTable(table=table.name, has_role=MEMBER_ROLE_COLUMN in (table.columns or {}))
return None
TRUNK_MERGE_QUEUE_SCHEMA = "MergeQueuePullRequests"
def resolve_trunk_merge_queue_table(team: Team, user_access_control: "UserAccessControl | None" = None) -> str | None:
"""The synced Trunk merge-queue table's warehouse name, or None.
The TrunkIo source leaves its merge-queue endpoint unselected by default (it needs a target
branch the flaky-tests endpoints don't), so absence is the normal state and every consumer
degrades to the GitHub-derived proxy. Team-level: a Trunk source covers one merge queue, so
the first synced table (oldest source first, for determinism) wins rather than threading repo
scope through a source that has none. ``user_access_control`` applies the requesting user's
per-source warehouse RBAC through the same scope the GitHub resolver uses
(``_accessible_sources``): a user denied every TrunkIo source resolves None, degrading to the
proxy exactly like the unsynced state.
"""
for source in _accessible_sources(team, ExternalDataSourceType.TRUNKIO, user_access_control):
table = _synced_table_name(team, source, TRUNK_MERGE_QUEUE_SCHEMA)
if table:
return table
return None
TRUNK_QUARANTINED_TESTS_SCHEMA = "QuarantinedTests"
@frozen
class TrunkQuarantineSource:
"""The synced Trunk quarantined-tests table plus the source's Trunk org slug (for app links)."""
table: str
org_url_slug: str | None
# False until a source moved to the v2 API resyncs, so the board keeps reading the v1 columns.
v2_shape: bool
def resolve_trunk_quarantined_tests_source(
team: Team, repository: str, user_access_control: "UserAccessControl | None" = None
) -> TrunkQuarantineSource | None:
"""The synced Trunk quarantined-tests table's warehouse name and org slug, or None.
A TrunkIo source is configured for one repository (``repo_owner``/``repo_name``), so prefer
the source matching ``repository`` and fall back to the oldest synced source only when none
declares a match (legacy sources without those keys). Per-user warehouse RBAC applies, and
None degrades the consumer to an honest ``available: false`` rather than an error.
"""
fallback: TrunkQuarantineSource | None = None
for source in _accessible_sources(team, ExternalDataSourceType.TRUNKIO, user_access_control):
for schema in _synced_schemas(team=team, source=source):
if schema.name != TRUNK_QUARANTINED_TESTS_SCHEMA:
continue
table = schema.table
if table is None or table.deleted or not _IDENTIFIER.match(table.name):
continue
# job_inputs is an EncryptedJSONField and can hold any JSON shape.
try:
inputs = SourceInputsStore.load(source)
except UnreadableSourceInputsError:
inputs = {}
resolved = TrunkQuarantineSource(
table=table.name,
org_url_slug=inputs.get("org_url_slug") or None,
v2_shape=trunk_quarantined_tests.V2_SHAPE_COLUMN in (table.columns or {}),
)
owner, name = inputs.get("repo_owner") or "", inputs.get("repo_name") or ""
if f"{owner}/{name}".lower() == repository.lower():
return resolved
# A source that names another repository holds that repository's quarantines, so only a
# legacy source with no repository keys may stand in.
if not owner and not name:
fallback = fallback or resolved
return fallback
# Listing the team's connected sources is its own concern (no curated read handle): it threads the
# requesting user's access control so the picker can't enumerate sources the user can't access.
def build_github_sources(*, team: Team, user_access_control: "UserAccessControl | None" = None) -> list[GitHubSource]:
return list_github_sources(team=team, user_access_control=user_access_control)
def list_github_sources(*, team: Team, user_access_control: "UserAccessControl | None" = None) -> list[GitHubSource]:
"""The team's selectable ``(source, repo)`` refs the caller may access, oldest source first.
One entry per repository a source is configured to sync, so a multi-repo source contributes one
entry per repo — the picker lists the handful of repos actually wired to the team's sources, not
the whole GitHub App's repo catalog. Configured (not just synced) repos are listed, including a
source still backfilling, so a just-connected source shows up; selecting a not-yet-synced repo
surfaces the same connect prompt ``resolve_github_tables`` drives. A source with no configured
repo still yields one blank-repo entry so it never vanishes. Sources the user can't access
(``user_access_control``) are filtered out. Each entry's ``id`` + ``repo`` are what the caller
passes back as ``source_id`` + ``repo`` to read that specific repo, and ``synced`` says whether that
repo has both endpoints the resolver needs — so the default (unscoped) page selects the first synced
entry and labels it with the repo the backend actually resolves, not a still-backfilling one listed
first.
"""
entries: list[GitHubSource] = []
for source in _github_sources(team, user_access_control):
by_repo = _synced_tables_by_repo(team=team, source=source)
synced_repos = {
repo
for repo, tables in by_repo.items()
if PULL_REQUESTS_SCHEMA in tables.names and WORKFLOW_RUNS_SCHEMA in tables.names
}
for repo in _configured_repositories(source) or [""]:
entries.append(
GitHubSource(
id=str(source.id),
repo=repo,
prefix=source.prefix or "",
synced=repo.casefold() in synced_repos,
)
)
return entries
class _RepoTables(NamedTuple):
# ``{endpoint: table name}`` for this one repo's synced curated schemas.
names: dict[str, str]
issue_events_team_requests: bool
workflow_runs_optional_columns: frozenset[str]
# ``{endpoint: last_synced_at}`` for the same schemas as ``names``.
synced_at: dict[str, datetime | None]
class _RepoCandidate(NamedTuple):
# Display repo: the source's original-case ``repository`` for its legacy/bare repo (``''`` when
# a bare row has no repo to attribute it to); the parsed ``owner/repo`` for a qualified repo.
repository: str
tables: _RepoTables
source_id: str
# True when the source lists this repo, so ``list_github_sources`` reports it as synced.
listed: bool
def _listed_first(candidates: Iterable[_RepoCandidate]) -> Iterator[_RepoCandidate]:
"""Listed candidates in source order, then the unlisted ones, without reading sources ahead."""
unlisted: list[_RepoCandidate] = []
for candidate in candidates:
if candidate.listed:
yield candidate
else:
unlisted.append(candidate)
yield from unlisted
def _repo_candidates(*, team: Team, sources: QuerySet[ExternalDataSource]) -> Iterator[_RepoCandidate]:
"""Every ``(repo, tables)`` a team's GitHub sources expose, in default-resolution order.
Sources keep their oldest-first order (the established connection wins), and within a source
repos follow the source's configured ``repositories`` order — the legacy/bare repo has no
inherent priority, so whichever repo the multi-select lists first is the default; unknown
leftovers trail, sorted. A multi-repo source contributes one entry
per repo, so a repo-scoped read can reach a repo that isn't the source's legacy one. Lazy (one
``ExternalDataSchema`` query per source, on demand) so the default resolve path stops at the
first usable repo instead of querying every source up front.
"""
for source in sources:
display = _source_repository(source)
legacy_key = display.casefold()
# Order strictly by the source's configured repo order, so the default (first) repo matches what
# the picker labels as githubSources[0]. The legacy/bare repo gets no priority here — if the
# `repositories` multi-select puts another repo first, that repo is the default on both sides;
# legacy_key is only used below to map the bare row's display name. Unknown leftovers trail.
configured = [repo.casefold() for repo in _configured_repositories(source)]
def order_key(repo_key: str, *, configured: list[str] = configured) -> tuple:
position = configured.index(repo_key) if repo_key in configured else len(configured)
return (position, repo_key)
by_repo = _synced_tables_by_repo(team=team, source=source)
for repo_key in sorted(by_repo, key=order_key):
repository = display if repo_key == legacy_key else repo_key
# Mirrors list_github_sources: a source with no configured repo lists one blank entry.
listed = repo_key in configured if configured else repo_key == ""
yield _RepoCandidate(
repository=repository, tables=by_repo[repo_key], source_id=str(source.id), listed=listed
)
def _github_sources(team: Team, user_access_control: "UserAccessControl | None" = None) -> QuerySet[ExternalDataSource]:
"""The team's non-deleted GitHub sources the caller may access, oldest first — the order
``resolve_github_tables`` defaults from, so a picker's first entry matches the default source.
Access scope comes from ``_accessible_sources``, shared with the Trunk resolver.
"""
return _accessible_sources(team, ExternalDataSourceType.GITHUB, user_access_control)
def _accessible_sources(
team: Team, source_type: ExternalDataSourceType, user_access_control: "UserAccessControl | None"
) -> QuerySet[ExternalDataSource]:
"""The team's non-deleted sources of ``source_type`` the caller may access.
``user_access_control`` applies the requesting user's per-source warehouse RBAC, so no resolver
or picker can reach a source the user can't access; ``None`` (system/Temporal/CLI contexts)
skips it, leaving team scoping. This is the single place that access scope is decided.
"""
sources = (
ExternalDataSource.objects.filter(team_id=team.pk, source_type=source_type)
.exclude(deleted=True)
.order_by("created_at", "id")
)
if user_access_control is not None:
sources = user_access_control.filter_queryset_by_access_level(sources)
return sources
# Distinct from the no-source message: the caller picked a source that isn't a usable GitHub
# source for this team (wrong id, another team's source, or its endpoints aren't synced).
_NO_SELECTED_SOURCE = "The selected GitHub source isn't connected or has no synced pull_requests/workflow_runs tables."
def _source_repository(source: ExternalDataSource) -> str:
"""The source's configured ``owner/repo`` identity, or '' when unset.
``job_inputs`` is an ``EncryptedJSONField`` that can hold any JSON value, so a non-dict
(list/str) would crash the ``.get`` below — guard it the same way ``job_logs.coordinator``
does, since this resolves for every engineering_analytics endpoint.
"""
try:
job_inputs = SourceInputsStore.load(source)
except UnreadableSourceInputsError:
return ""
return str(job_inputs.get("repository") or "").strip()
def _configured_repositories(source: ExternalDataSource) -> list[str]:
"""The ``owner/repo`` names a source is configured to sync, in order, original case, deduped.
Reads the multi-repo ``repositories`` list, falling back to the legacy single ``repository`` —
the same precedence the source-side ``effective_repositories`` parser uses, but tolerant (never
raises) since this only drives a display picker. Case is preserved for the label; duplicates are
dropped case-insensitively (GitHub full names are case-insensitive). ``job_inputs`` is an
``EncryptedJSONField`` that can hold any JSON value, so a non-dict yields no repos.
"""
try:
job_inputs = SourceInputsStore.load(source)
except UnreadableSourceInputsError:
return []
repositories = job_inputs.get("repositories")
# An empty `repositories` list means "unset" — the sync-side parser falls back to the single
# `repository` and still syncs it, so treat it the same here or a working single-repo source
# would list as a blank, unsynced entry.
raw = repositories if isinstance(repositories, list) and repositories else [job_inputs.get("repository")]
seen: set[str] = set()
result: list[str] = []
for value in raw:
name = str(value or "").strip()
key = name.casefold()
if name and key not in seen:
seen.add(key)
result.append(name)
return result
def _as_source_uuid(source_id: str) -> UUID:
try:
return UUID(source_id)
except ValueError as err:
raise ValueError(f"source_id must be a UUID, got: {source_id!r}") from err
def resolve_depot_job_attempts_tables(
team: Team, user_access_control: "UserAccessControl | None" = None
) -> dict[str, depot_ci.DepotJobAttempts]:
"""The team's synced Depot CI job attempts, keyed by the casefolded ``owner/repo`` the way
``_synced_tables_by_repo`` keys GitHub repos.
A Depot source syncs only the repository its ``repository`` input names, so that input decides
which GitHub repository its runs join. The oldest source wins, like every other resolver here.
"""
return {
repository: depot.attempts
for repository, depot in _depot_job_attempts_by_repo(team, user_access_control).items()
}
@frozen
class _SyncedDepotJobAttempts:
attempts: depot_ci.DepotJobAttempts
synced_at: datetime | None
def _depot_job_attempts_by_repo(
team: Team, user_access_control: "UserAccessControl | None"
) -> dict[str, _SyncedDepotJobAttempts]:
by_repo: dict[str, _SyncedDepotJobAttempts] = {}
for source in _accessible_sources(team, ExternalDataSourceType.DEPOT, user_access_control):
repository = _source_repository(source).casefold()
schema = _synced_schema(team, source, DEPOT_JOB_ATTEMPTS_SCHEMA) if repository else None
table = schema.table if schema is not None else None
attempts = depot_ci.DepotJobAttempts.for_repository(table.name, repository) if table is not None else None
if schema is not None and attempts is not None:
by_repo.setdefault(repository, _SyncedDepotJobAttempts(attempts=attempts, synced_at=schema.last_synced_at))
return by_repo
def resolve_ci_data_freshness(
*, team: Team, tables: GitHubTables, user_access_control: "UserAccessControl | None" = None
) -> CIDataFreshness:
"""When the CI data that ``tables`` reads was last synced from its source.
Runs and jobs of a repository with synced Depot CI job attempts come from two tables each, so each
time is the older of the two. Every row a read returns is then at least that fresh.
"""
depot = _depot_job_attempts_by_repo(team, user_access_control).get(tables.repository.casefold())
def watermark(github_synced_at: datetime | None) -> datetime | None:
if depot is None:
return github_synced_at
if github_synced_at is None or depot.synced_at is None:
return None
return min(github_synced_at, depot.synced_at)
return CIDataFreshness(
runs_synced_at=watermark(tables.workflow_runs_synced_at),
jobs_synced_at=watermark(tables.workflow_jobs_synced_at) if tables.workflow_jobs else None,
)
def depot_source_api_token(team: Team, source_id: str) -> str | None:
"""The Depot organization API token a team's Depot source syncs with, or None."""
source = _accessible_sources(team, ExternalDataSourceType.DEPOT, None).filter(id=source_id).first()
# job_inputs is an EncryptedJSONField and can hold any JSON shape.
try:
inputs = SourceInputsStore.load(source) if source is not None else {}
except UnreadableSourceInputsError:
inputs = {}
api_token = inputs.get("api_token")
return api_token if isinstance(api_token, str) and api_token else None
def depot_source_job_attempts_table(team: Team, source: ExternalDataSource) -> str | None:
"""The synced ``job_attempts`` table of one Depot source, or None."""
return _synced_table_name(team, source, DEPOT_JOB_ATTEMPTS_SCHEMA)
def depot_source_job_attempts(team: Team, source: ExternalDataSource) -> depot_ci.DepotJobAttempts | None:
"""The synced job attempts of one Depot source, scoped to the repository it syncs, or None."""
table = depot_source_job_attempts_table(team, source)
repository = _source_repository(source).casefold()
return depot_ci.DepotJobAttempts.for_repository(table, repository) if table and repository else None
def _synced_table_name(team: Team, source: ExternalDataSource, schema_name: str) -> str | None:
"""The warehouse table a source syncs for ``schema_name``, or None. Only a plain identifier
qualifies, because callers interpolate the name into HogQL."""
schema = _synced_schema(team, source, schema_name)
return schema.table.name if schema is not None and schema.table is not None else None
def _synced_schema(team: Team, source: ExternalDataSource, schema_name: str) -> ExternalDataSchema | None:
for schema in _synced_schemas(team=team, source=source):
table = schema.table
if schema.name == schema_name and table is not None and not table.deleted and _IDENTIFIER.match(table.name):
return schema
return None
def _synced_schemas(*, team: Team, source: ExternalDataSource) -> QuerySet[ExternalDataSchema]:
return (
ExternalDataSchema.objects.filter(team_id=team.pk, source_id=source.id, should_sync=True)
.exclude(deleted=True)
.select_related("table")
)
def _synced_tables_by_repo(*, team: Team, source: ExternalDataSource) -> dict[str, _RepoTables]:
"""Map ``{repository: tables}`` for a source's actively-synced curated schemas.
A source syncs one repo (legacy bare endpoint names like ``pull_requests``) or several
(repo-qualified names like ``owner/repo.pull_requests``, the multi-repo GitHub source). Each
row resolves to its ``(repository, endpoint)`` through the warehouse_sources facade — persisted
metadata first, qualified-name parse second, the source's legacy ``repository`` for bare rows
last. Repo keys are normalized (lowercased, since GitHub full names are case-insensitive); a
bare row with no legacy repo groups under ``''`` — the pre-multi-repo single-repo shape.
"""
legacy_repo = _source_repository(source) or None
by_repo: dict[str, dict[str, str]] = {}
team_requests_by_repo: dict[str, bool] = {}
run_columns_by_repo: dict[str, frozenset[str]] = {}
synced_at_by_repo: dict[str, dict[str, datetime | None]] = {}
for schema in _synced_schemas(team=team, source=source):
repository, endpoint = github_schema_repo_endpoint(schema.schema_metadata, schema.name, legacy_repo)
if endpoint not in _CURATED_ENDPOINTS:
continue
table = schema.table
if table is not None and not table.deleted and _IDENTIFIER.match(table.name):
by_repo.setdefault(repository or "", {})[endpoint] = table.name
# Set together with the table name, so a later schema row for the same endpoint replaces all of them.
synced_at_by_repo.setdefault(repository or "", {})[endpoint] = schema.last_synced_at
if endpoint == ISSUE_EVENTS_SCHEMA:
team_requests_by_repo[repository or ""] = REQUESTED_TEAM_COLUMN in (table.columns or {})
if endpoint == WORKFLOW_RUNS_SCHEMA:
run_columns_by_repo[repository or ""] = frozenset(OPTIONAL_WORKFLOW_RUNS_COLUMNS) & frozenset(
table.columns or {}
)
return {
repository: _RepoTables(
names=names,
issue_events_team_requests=team_requests_by_repo.get(repository, False),
workflow_runs_optional_columns=run_columns_by_repo.get(repository, frozenset()),
synced_at=synced_at_by_repo.get(repository, {}),
)
for repository, names in by_repo.items()
}