Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(signals): split implementation pr report lookup into indexed queries (#125960)

Dylan Martindb29f481b996
diff --git a/products/signals/backend/implementation_pr.py b/products/signals/backend/implementation_pr.py
index 11f88f46f278..e51c2ff68d88 100644
--- a/products/signals/backend/implementation_pr.py
+++ b/products/signals/backend/implementation_pr.py
@@ -334,28 +334,44 @@ def fetch_implementation_pr_urls_for_reports(report_ids: list[str], *, team_id:
def report_ids_for_implementation_pr(*, team_id: int, repository: str, pr_number: int) -> list[str]:
+ # The webhook reaches every team on the installation. A team without reports must not pay for
+ # the regex scan over its task runs below.
+ if not SignalReport.objects.filter(team_id=team_id).exists():
+ return []
# Narrow by task output before resolving reports so webhooks do not scan the whole inbox.
owner, repo = repository.split("/", 1)
url_body = rf'[^:"]+://(www\.)?github\.com/+{re.escape(owner)}/+{re.escape(repo)}/+pull/+0*{pr_number}'
url_pattern = rf"^{url_body}([/?#]|$)"
array_url_pattern = rf'"{url_body}([/?#"]|$)'
- task_ids = tasks_facade.task_ids_with_pr_url_subquery(
- team_id,
- pr_bearing_task_run_filter(),
- Q(output__pr_url__iregex=url_pattern) | Q(output__pr_urls__iregex=array_url_pattern),
- )
- candidates = SignalReport.objects.filter(team_id=team_id).filter(
- Q(
- artefacts__pull_request__repository__iexact=repository,
- artefacts__pull_request__number=pr_number,
- artefacts__pull_request__team_id=team_id,
- )
- | Q(assignment__repository__iexact=repository, assignment__pr_number=pr_number)
- | SignalReport.reports_for_task_ids_filter(task_ids, team_id=team_id)
- )
- prs = fetch_implementation_prs_for_reports(
- [str(report_id) for report_id in candidates.values_list("id", flat=True).distinct()], team_id=team_id
+ task_ids = list(
+ tasks_facade.task_ids_with_pr_url_subquery(
+ team_id,
+ pr_bearing_task_run_filter(),
+ Q(output__pr_url__iregex=url_pattern) | Q(output__pr_urls__iregex=array_url_pattern),
+ ).values_list("task_id", flat=True)
)
+ # One `OR` over joins to every link table defeats the indexes, so each path is its own lookup.
+ candidate_querysets = [
+ # The array form, as in `implementation_pr_report_filter`, so Postgres probes the artefact index.
+ SignalReportArtefact.objects.filter(
+ _EqualsAny(
+ F("pull_request_id"),
+ ArraySubquery(
+ SignalReportPullRequest.objects.for_team(team_id)
+ .filter(repository__iexact=repository, number=pr_number)
+ .values("id")
+ ),
+ ),
+ team_id=team_id,
+ ).values("report_id"),
+ SignalReportAssignment.all_teams.filter(
+ team_id=team_id, repository__iexact=repository, pr_number=pr_number
+ ).values("report_id"),
+ ]
+ if task_ids:
+ candidate_querysets.extend(SignalReport.report_ids_for_task_ids(task_ids, team_id=team_id))
+ candidate_ids = {str(row["report_id"]) for queryset in candidate_querysets for row in queryset}
+ prs = fetch_implementation_prs_for_reports(sorted(candidate_ids), team_id=team_id)
return [
report_id
for report_id, report_prs in prs.items()
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 52f8818cf11a..a203e24ef8a3 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -1124,6 +1124,13 @@ class SignalReport(UUIDModel):
can invert the join and scan every team's `task_run` artefacts when `task_ids` is a
subquery. Only cross-team callers with literal `task_ids` (the PR webhook) leave it unset.
"""
+ artefact_rows, legacy_rows, assignment_rows = SignalReport.report_ids_for_task_ids(task_ids, team_id=team_id)
+ return models.Q(id__in=artefact_rows) | models.Q(id__in=legacy_rows) | models.Q(id__in=assignment_rows)
+
+ @staticmethod
+ def report_ids_for_task_ids(task_ids: Any, *, team_id: int | None = None) -> list["models.QuerySet[Any]"]:
+ """The `report_id` querysets behind `reports_for_task_ids_filter`, one per link table. A caller
+ can run them as separate indexed lookups instead of one `OR` over `SignalReport`."""
artefact_rows = SignalReportArtefact.objects.filter(
type=SignalReportArtefact.ArtefactType.TASK_RUN, task_id__in=task_ids
)
@@ -1136,11 +1143,7 @@ class SignalReport(UUIDModel):
artefact_rows = artefact_rows.filter(team_id=team_id)
legacy_rows = legacy_rows.filter(team_id=team_id)
assignment_rows = assignment_rows.filter(team_id=team_id)
- return (
- models.Q(id__in=artefact_rows.values("report_id"))
- | models.Q(id__in=legacy_rows.values("report_id"))
- | models.Q(id__in=assignment_rows.values("report_id"))
- )
+ return [rows.values("report_id") for rows in (artefact_rows, legacy_rows, assignment_rows)]
class SignalReportAssignment(TeamScopedRootMixin, UUIDModel):
diff --git a/products/signals/backend/test/test_implementation_pr_cache.py b/products/signals/backend/test/test_implementation_pr_cache.py
index 4549a0ff2347..606ebd7cedd1 100644
--- a/products/signals/backend/test/test_implementation_pr_cache.py
+++ b/products/signals/backend/test/test_implementation_pr_cache.py
@@ -8,8 +8,10 @@ from django.db import OperationalError
from parameterized import parameterized
+from products.signals.backend.implementation_pr import report_ids_for_implementation_pr
from products.signals.backend.implementation_pr_cache import live_implementation_pr_flags
from products.signals.backend.models import (
+ SignalActorKind,
SignalReport,
SignalReportArtefact,
SignalReportAssignment,
@@ -96,6 +98,12 @@ class TestImplementationPrCache(BaseTest):
task, _ = self._task(with_pr=True)
SignalReportTask.objects.create(team=self.team, report=self.report, task=task, relationship="implementation")
+ def _link_task_assignment(self) -> None:
+ task, _ = self._task(with_pr=True)
+ SignalReportAssignment.all_teams.create(
+ team=self.team, report=self.report, actor_kind=SignalActorKind.TASK, actor_task_id=task.id
+ )
+
@parameterized.expand(
[
("assignment", "_link_assignment"),
@@ -114,6 +122,30 @@ class TestImplementationPrCache(BaseTest):
assert live == (True, True)
assert cached == live
+ @parameterized.expand(
+ [
+ ("assignment", "_link_assignment"),
+ ("pull_request_artefact", "_link_pull_request"),
+ ("task_run_artefact", "_link_task_run_artefact"),
+ ("legacy_report_task", "_link_legacy_report_task"),
+ ("task_assignment", "_link_task_assignment"),
+ ]
+ )
+ def test_each_link_source_resolves_the_report_for_its_pull_request(self, _name: str, link: str) -> None:
+ getattr(self, link)()
+
+ assert report_ids_for_implementation_pr(team_id=self.team.id, repository="Example/App", pr_number=7) == [
+ str(self.report.id)
+ ]
+ assert report_ids_for_implementation_pr(team_id=self.team.id, repository="example/app", pr_number=8) == []
+
+ def test_a_team_without_reports_skips_the_task_run_scan(self) -> None:
+ self._task(with_pr=True)
+ self.report.delete()
+
+ with self.assertNumQueries(1):
+ assert report_ids_for_implementation_pr(team_id=self.team.id, repository="example/app", pr_number=7) == []
+
def test_a_merged_pull_request_stops_counting_as_active(self) -> None:
with self.captureOnCommitCallbacks(execute=True):
self._link_pull_request()