:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(signals): split implementation pr report lookup into indexed queries (#125960)
Dylan Martindb29f481b996
diff --git a/products/signals/backend/implementation_pr.py b/products/signals/backend/implementation_pr.pyindex 11f88f46f278..e51c2ff68d88 100644--- a/products/signals/backend/implementation_pr.py+++ b/products/signals/backend/implementation_pr.py@@ -334,28 +334,44 @@ def fetch_implementation_pr_urls_for_reports(report_ids: list[str], *, team_id:def report_ids_for_implementation_pr(*, team_id: int, repository: str, pr_number: int) -> list[str]:+ # The webhook reaches every team on the installation. A team without reports must not pay for+ # the regex scan over its task runs below.+ if not SignalReport.objects.filter(team_id=team_id).exists():+ return []# Narrow by task output before resolving reports so webhooks do not scan the whole inbox.owner, repo = repository.split("/", 1)url_body = rf'[^:"]+://(www\.)?github\.com/+{re.escape(owner)}/+{re.escape(repo)}/+pull/+0*{pr_number}'url_pattern = rf"^{url_body}([/?#]|$)"array_url_pattern = rf'"{url_body}([/?#"]|$)'- task_ids = tasks_facade.task_ids_with_pr_url_subquery(- team_id,- pr_bearing_task_run_filter(),- Q(output__pr_url__iregex=url_pattern) | Q(output__pr_urls__iregex=array_url_pattern),- )- candidates = SignalReport.objects.filter(team_id=team_id).filter(- Q(- artefacts__pull_request__repository__iexact=repository,- artefacts__pull_request__number=pr_number,- artefacts__pull_request__team_id=team_id,- )- | Q(assignment__repository__iexact=repository, assignment__pr_number=pr_number)- | SignalReport.reports_for_task_ids_filter(task_ids, team_id=team_id)- )- prs = fetch_implementation_prs_for_reports(- [str(report_id) for report_id in candidates.values_list("id", flat=True).distinct()], team_id=team_id+ task_ids = list(+ tasks_facade.task_ids_with_pr_url_subquery(+ team_id,+ pr_bearing_task_run_filter(),+ Q(output__pr_url__iregex=url_pattern) | Q(output__pr_urls__iregex=array_url_pattern),+ ).values_list("task_id", flat=True))+ # One `OR` over joins to every link table defeats the indexes, so each path is its own lookup.+ candidate_querysets = [+ # The array form, as in `implementation_pr_report_filter`, so Postgres probes the artefact index.+ SignalReportArtefact.objects.filter(+ _EqualsAny(+ F("pull_request_id"),+ ArraySubquery(+ SignalReportPullRequest.objects.for_team(team_id)+ .filter(repository__iexact=repository, number=pr_number)+ .values("id")+ ),+ ),+ team_id=team_id,+ ).values("report_id"),+ SignalReportAssignment.all_teams.filter(+ team_id=team_id, repository__iexact=repository, pr_number=pr_number+ ).values("report_id"),+ ]+ if task_ids:+ candidate_querysets.extend(SignalReport.report_ids_for_task_ids(task_ids, team_id=team_id))+ candidate_ids = {str(row["report_id"]) for queryset in candidate_querysets for row in queryset}+ prs = fetch_implementation_prs_for_reports(sorted(candidate_ids), team_id=team_id)return [report_idfor report_id, report_prs in prs.items()diff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 52f8818cf11a..a203e24ef8a3 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -1124,6 +1124,13 @@ class SignalReport(UUIDModel):can invert the join and scan every team's `task_run` artefacts when `task_ids` is asubquery. Only cross-team callers with literal `task_ids` (the PR webhook) leave it unset."""+ artefact_rows, legacy_rows, assignment_rows = SignalReport.report_ids_for_task_ids(task_ids, team_id=team_id)+ return models.Q(id__in=artefact_rows) | models.Q(id__in=legacy_rows) | models.Q(id__in=assignment_rows)++ @staticmethod+ def report_ids_for_task_ids(task_ids: Any, *, team_id: int | None = None) -> list["models.QuerySet[Any]"]:+ """The `report_id` querysets behind `reports_for_task_ids_filter`, one per link table. A caller+ can run them as separate indexed lookups instead of one `OR` over `SignalReport`."""artefact_rows = SignalReportArtefact.objects.filter(type=SignalReportArtefact.ArtefactType.TASK_RUN, task_id__in=task_ids)@@ -1136,11 +1143,7 @@ class SignalReport(UUIDModel):artefact_rows = artefact_rows.filter(team_id=team_id)legacy_rows = legacy_rows.filter(team_id=team_id)assignment_rows = assignment_rows.filter(team_id=team_id)- return (- models.Q(id__in=artefact_rows.values("report_id"))- | models.Q(id__in=legacy_rows.values("report_id"))- | models.Q(id__in=assignment_rows.values("report_id"))- )+ return [rows.values("report_id") for rows in (artefact_rows, legacy_rows, assignment_rows)]class SignalReportAssignment(TeamScopedRootMixin, UUIDModel):diff --git a/products/signals/backend/test/test_implementation_pr_cache.py b/products/signals/backend/test/test_implementation_pr_cache.pyindex 4549a0ff2347..606ebd7cedd1 100644--- a/products/signals/backend/test/test_implementation_pr_cache.py+++ b/products/signals/backend/test/test_implementation_pr_cache.py@@ -8,8 +8,10 @@ from django.db import OperationalErrorfrom parameterized import parameterized+from products.signals.backend.implementation_pr import report_ids_for_implementation_prfrom products.signals.backend.implementation_pr_cache import live_implementation_pr_flagsfrom products.signals.backend.models import (+ SignalActorKind,SignalReport,SignalReportArtefact,SignalReportAssignment,@@ -96,6 +98,12 @@ class TestImplementationPrCache(BaseTest):task, _ = self._task(with_pr=True)SignalReportTask.objects.create(team=self.team, report=self.report, task=task, relationship="implementation")+ def _link_task_assignment(self) -> None:+ task, _ = self._task(with_pr=True)+ SignalReportAssignment.all_teams.create(+ team=self.team, report=self.report, actor_kind=SignalActorKind.TASK, actor_task_id=task.id+ )+@parameterized.expand([("assignment", "_link_assignment"),@@ -114,6 +122,30 @@ class TestImplementationPrCache(BaseTest):assert live == (True, True)assert cached == live+ @parameterized.expand(+ [+ ("assignment", "_link_assignment"),+ ("pull_request_artefact", "_link_pull_request"),+ ("task_run_artefact", "_link_task_run_artefact"),+ ("legacy_report_task", "_link_legacy_report_task"),+ ("task_assignment", "_link_task_assignment"),+ ]+ )+ def test_each_link_source_resolves_the_report_for_its_pull_request(self, _name: str, link: str) -> None:+ getattr(self, link)()++ assert report_ids_for_implementation_pr(team_id=self.team.id, repository="Example/App", pr_number=7) == [+ str(self.report.id)+ ]+ assert report_ids_for_implementation_pr(team_id=self.team.id, repository="example/app", pr_number=8) == []++ def test_a_team_without_reports_skips_the_task_run_scan(self) -> None:+ self._task(with_pr=True)+ self.report.delete()++ with self.assertNumQueries(1):+ assert report_ids_for_implementation_pr(team_id=self.team.id, repository="example/app", pr_number=7) == []+def test_a_merged_pull_request_stops_counting_as_active(self) -> None:with self.captureOnCommitCallbacks(execute=True):self._link_pull_request()