:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): cache implementation PR state on reports (#124388)
Dylan Martin09d34bf4b38d
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txtindex bb0609ecf09b..d776d5d1b54c 100644--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt@@ -116,6 +116,8 @@ post_delete:products.feature_flags.backend.local_evaluation.feature_flag_changedpost_delete:products.managed_warehouse.backend.models._observe_duckgres_server_deletepost_delete:products.replay_vision.backend.media_expiry.expire_media_assetpost_delete:products.review_hog.backend.ownership._invalidate_owned_repositories_on_delete+post_delete:products.signals.backend.implementation_pr_cache._refresh_for_artefact+post_delete:products.signals.backend.implementation_pr_cache._refresh_for_report_linkpost_delete:products.signals.backend.receivers.mark_scout_suggestions_stale_on_fleet_changepost_delete:products.signals.backend.receivers.reconcile_report_embedding_on_verdict_deletedpost_delete:products.signals.backend.receivers.sync_report_latest_actionability_on_delete@@ -240,6 +242,10 @@ post_save:products.metrics.backend.dashboard_import.receivers.schedule_import_fipost_save:products.posthog_ai.backend.receivers.mark_conversation_sandbox_on_first_runpost_save:products.review_hog.backend.ownership._invalidate_owned_repositories_on_savepost_save:products.review_hog.backend.receivers.handle_task_run_saved+post_save:products.signals.backend.implementation_pr_cache._refresh_for_artefact+post_save:products.signals.backend.implementation_pr_cache._refresh_for_pull_request+post_save:products.signals.backend.implementation_pr_cache._refresh_for_report_link+post_save:products.signals.backend.implementation_pr_cache._refresh_for_task_runpost_save:products.signals.backend.receivers.arm_pending_checks_when_report_monitoringpost_save:products.signals.backend.receivers.capture_status_change_analyticspost_save:products.signals.backend.receivers.close_pr_when_report_dismisseddiff --git a/products/signals/backend/apps.py b/products/signals/backend/apps.pyindex 1b18a761959c..57fc582ca7ad 100644--- a/products/signals/backend/apps.py+++ b/products/signals/backend/apps.py@@ -20,10 +20,12 @@ class SignalsConfig(AppConfig):# receivers: post_save receiver that closes a report's implementation PR on suppression/snooze.from . import (activity_logging, # noqa: F401+ implementation_pr_cache,receivers, # noqa: F401)receivers.connect_task_run_assignment_sync()+ implementation_pr_cache.connect_implementation_pr_cache_receivers()self._register_task_spend_updates()self._register_signal_emission_gate()self._register_scout_skill_archive_guard()diff --git a/products/signals/backend/implementation_pr.py b/products/signals/backend/implementation_pr.pyindex d7c888b06028..7eed6fd49674 100644--- a/products/signals/backend/implementation_pr.py+++ b/products/signals/backend/implementation_pr.py@@ -1,14 +1,14 @@"""Resolve implementation PR URLs linked to signal reports."""import re-from collections.abc import Callable+from collections.abc import Callable, Sequencefrom dataclasses import replacefrom datetime import datetime-from typing import TYPE_CHECKING, Any, Literal+from typing import TYPE_CHECKING, Any, Literal, TypeVarfrom uuid import NAMESPACE_URL, uuid5from django.contrib.postgres.expressions import ArraySubquery-from django.db.models import F, Lookup, Q+from django.db.models import F, Lookup, Q, QuerySetimport structlog@@ -38,6 +38,8 @@ from products.tasks.backend.facade import api as tasks_facadelogger = structlog.get_logger(__name__)+_LinkModel = TypeVar("_LinkModel", SignalReportArtefact, SignalReportAssignment, SignalReportTask)+if TYPE_CHECKING:from django.db.backends.base.base import BaseDatabaseWrapperfrom django.db.models.sql.compiler import SQLCompiler@@ -67,34 +69,69 @@ class _EqualsAny(Lookup):return f"{lhs} = ANY({rhs})", (*lhs_params, *rhs_params)-def implementation_pr_report_filter(*, team_id: int, active_only: bool = False) -> Q:+def implementation_pr_report_filter(+ *, team_id: int, active_only: bool = False, report_ids: Sequence[Any] | None = None+) -> Q:+ """Matches the reports with an implementation PR.++ `report_ids` narrows every link subquery to those reports, so a caller that asks about a few+ reports does not scan the team's links and PR-bearing task runs. The answer for those reports+ is the same either way.+ """+# Every branch is a team-scoped subquery on the report id. A join through `assignment` or# `pull_request` lets the planner scan every team's rows before it applies the team scope.- assignments = SignalReportAssignment.all_teams.filter(team_id=team_id, pr_url__regex=_PULL_REQUEST_URL_PATTERN)+ def for_reports(queryset: QuerySet[_LinkModel]) -> QuerySet[_LinkModel]:+ return queryset if report_ids is None else queryset.filter(report_id__in=report_ids)++ assignments = for_reports(+ SignalReportAssignment.all_teams.filter(team_id=team_id, pr_url__regex=_PULL_REQUEST_URL_PATTERN)+ )pull_requests = SignalReportPullRequest.objects.for_team(team_id).filter(url__regex=_PULL_REQUEST_URL_PATTERN)- task_ids = tasks_facade.task_ids_with_pr_url_subquery(- team_id,+ task_conditions = [pr_bearing_task_run_filter(),Q(output__pr_url__regex=_PULL_REQUEST_URL_PATTERN) | Q(output__pr_urls__regex=_PULL_REQUEST_URL_ARRAY_PATTERN),- )+ ]+ if report_ids is not None:+ pull_requests = pull_requests.filter(+ id__in=SignalReportArtefact.objects.filter(+ team_id=team_id, type=SignalReportArtefact.ArtefactType.PULL_REQUEST, report_id__in=report_ids+ ).values("pull_request_id")+ )+ task_conditions.append(+ Q(task_id__in=for_reports(SignalReportArtefact.objects.filter(team_id=team_id)).values("task_id"))+ | Q(task_id__in=for_reports(SignalReportTask.objects.filter(team_id=team_id)).values("task_id"))+ | Q(+ task_id__in=for_reports(SignalReportAssignment.all_teams.filter(team_id=team_id)).values(+ "actor_task_id"+ )+ )+ )+ task_ids = tasks_facade.task_ids_with_pr_url_subquery(team_id, *task_conditions)task_run_artefacts = (- SignalReportArtefact.objects.filter(- team_id=team_id,- type=SignalReportArtefact.ArtefactType.TASK_RUN,- task_id__in=task_ids,- content__regex=rf'"product"\s*:\s*"{SIGNALS_PRODUCT}"',+ for_reports(+ SignalReportArtefact.objects.filter(+ team_id=team_id,+ type=SignalReportArtefact.ArtefactType.TASK_RUN,+ task_id__in=task_ids,+ content__regex=rf'"product"\s*:\s*"{SIGNALS_PRODUCT}"',+ )).exclude(content__regex=rf'"type"\s*:\s*"({"|".join(sorted(NON_PR_BEARING_TASK_RUN_TYPES))})"').values("report_id"))- legacy_tasks = SignalReportTask.objects.filter(- team_id=team_id,- task_id__in=task_ids,- relationship__in=_PR_BEARING_LEGACY_TASK_RELATIONSHIPS,+ legacy_tasks = for_reports(+ SignalReportTask.objects.filter(+ team_id=team_id,+ task_id__in=task_ids,+ relationship__in=_PR_BEARING_LEGACY_TASK_RELATIONSHIPS,+ )).values("report_id")- assignment_tasks = SignalReportAssignment.all_teams.filter(- team_id=team_id,- actor_task_id__in=task_ids,+ assignment_tasks = for_reports(+ SignalReportAssignment.all_teams.filter(+ team_id=team_id,+ actor_task_id__in=task_ids,+ )).values("report_id")if active_only:@@ -110,10 +147,12 @@ def implementation_pr_report_filter(*, team_id: int, active_only: bool = False)# `pull_request_id IN (subquery)` keeps the cross-team merge join. Only the array form makes# Postgres probe `signals_artefact_pr_report_idx` with this team's PR ids.- pull_request_links = SignalReportArtefact.objects.filter(- _EqualsAny(F("pull_request_id"), ArraySubquery(pull_requests.values("id"))),- team_id=team_id,- type=SignalReportArtefact.ArtefactType.PULL_REQUEST,+ pull_request_links = for_reports(+ SignalReportArtefact.objects.filter(+ _EqualsAny(F("pull_request_id"), ArraySubquery(pull_requests.values("id"))),+ team_id=team_id,+ type=SignalReportArtefact.ArtefactType.PULL_REQUEST,+ )).values("report_id")return (diff --git a/products/signals/backend/implementation_pr_cache.py b/products/signals/backend/implementation_pr_cache.pynew file mode 100644index 000000000000..dd158538f0fd--- /dev/null+++ b/products/signals/backend/implementation_pr_cache.py@@ -0,0 +1,255 @@+"""Keep `SignalReport.has_implementation_pr` and `has_active_implementation_pr` equal to the link tables.++`implementation_pr_report_filter` is the source of truth. It reads five link sources, so the inbox+cannot afford to evaluate it for every report on each list and count. These helpers store its+answer on the report row instead. Receivers call `schedule_implementation_pr_cache_refresh` after a+link write, and the daily reconcile corrects rows that a write path outside the receivers left stale.+"""++from collections import defaultdict+from collections.abc import Iterable, Iterator, Sequence+from typing import Any++from django.db import transaction+from django.db.models import BooleanField, ExpressionWrapper, QuerySet+from django.db.models.signals import post_delete, post_save++import structlog++from posthog.dataclasses import frozen+from posthog.models.organization import Organization+from posthog.models.project import Project+from posthog.models.team import Team++from products.signals.backend.models import (+ SignalReport,+ SignalReportArtefact,+ SignalReportAssignment,+ SignalReportPullRequest,+ SignalReportTask,+)+from products.signals.backend.report_actionability_repair import RepairedBatch, report_id_batches+from products.tasks.backend.facade.task_run_signals import connect_task_run_post_save++logger = structlog.get_logger(__name__)++_LINK_ARTEFACT_TYPES = frozenset(+ {SignalReportArtefact.ArtefactType.PULL_REQUEST, SignalReportArtefact.ArtefactType.TASK_RUN}+)+# A delete that starts at one of these removes the linked reports too, so their links need no refresh.+_REPORT_REMOVING_ORIGINS = (SignalReport, Team, Project, Organization)+++@frozen+class LiveImplementationPrFlags:+ has_pr: ExpressionWrapper+ has_active_pr: ExpressionWrapper+++def live_implementation_pr_flags(team_id: int, report_ids: Sequence[Any] | None = None) -> LiveImplementationPrFlags:+ """The source-of-truth filter as two per-report annotations: any PR, and an active PR."""+ from products.signals.backend.implementation_pr import ( # noqa: PLC0415 — keeps the tasks facade off the Django setup path, which imports this module for its receivers+ implementation_pr_report_filter,+ )++ return LiveImplementationPrFlags(+ has_pr=ExpressionWrapper(+ implementation_pr_report_filter(team_id=team_id, report_ids=report_ids), output_field=BooleanField()+ ),+ has_active_pr=ExpressionWrapper(+ implementation_pr_report_filter(team_id=team_id, active_only=True, report_ids=report_ids),+ output_field=BooleanField(),+ ),+ )+++def refresh_implementation_pr_cache(*, team_id: int, report_ids: Iterable[Any]) -> int:+ """Recompute the cached flags of these reports and store the ones that changed.++ Returns how many reports changed. The rows stay locked from the read to the write, so a refresh+ that read the links before a concurrent link write committed cannot overwrite the newer answer.+ """+ ids = sorted({str(report_id) for report_id in report_ids})+ if not ids:+ return 0+ with transaction.atomic():+ # FOR NO KEY UPDATE, for the reason `refresh_latest_actionability` gives: link inserts hold+ # KEY SHARE on the report through their foreign key.+ locked = list(+ SignalReport.objects.select_for_update(no_key=True)+ .filter(team_id=team_id, id__in=ids)+ .order_by("id")+ .values_list("id", flat=True)+ )+ if not locked:+ return 0+ live = live_implementation_pr_flags(team_id, report_ids=locked)+ rows = (+ SignalReport.objects.filter(team_id=team_id, id__in=locked)+ .annotate(live_has_implementation_pr=live.has_pr, live_has_active_implementation_pr=live.has_active_pr)+ .values_list(+ "id",+ "has_implementation_pr",+ "has_active_implementation_pr",+ "live_has_implementation_pr",+ "live_has_active_implementation_pr",+ )+ )+ changed: dict[tuple[bool, bool], list[Any]] = defaultdict(list)+ for report_id, cached_has, cached_active, has_pr, has_active_pr in rows:+ if (cached_has, cached_active) != (has_pr, has_active_pr):+ changed[(has_pr, has_active_pr)].append(report_id)+ for (has_pr, has_active_pr), changed_ids in changed.items():+ # `update()`, not `save()`: a cache refresh must not bump `updated_at`, which the inbox+ # sorts on, or fire the report's save receivers.+ SignalReport.objects.filter(team_id=team_id, id__in=changed_ids).update(+ has_implementation_pr=has_pr, has_active_implementation_pr=has_active_pr+ )+ return sum(len(changed_ids) for changed_ids in changed.values())+++def refresh_implementation_pr_cache_for_links(+ *,+ team_id: int,+ report_ids: Iterable[Any] = (),+ pull_request_ids: Iterable[Any] = (),+ task_ids: Iterable[Any] = (),+) -> int:+ """Refresh the reports these links reach: the reports themselves, the reports a pull request+ links to, and the reports a task links to."""+ ids = {str(report_id) for report_id in report_ids}+ pull_request_ids = list(pull_request_ids)+ task_ids = list(task_ids)+ if pull_request_ids:+ ids.update(+ str(report_id)+ for report_id in SignalReportArtefact.objects.filter(+ team_id=team_id,+ type=SignalReportArtefact.ArtefactType.PULL_REQUEST,+ pull_request_id__in=pull_request_ids,+ ).values_list("report_id", flat=True)+ )+ if task_ids:+ ids.update(+ str(report_id)+ for report_id in SignalReportArtefact.objects.filter(+ team_id=team_id, type=SignalReportArtefact.ArtefactType.TASK_RUN, task_id__in=task_ids+ ).values_list("report_id", flat=True)+ )+ ids.update(+ str(report_id)+ for report_id in SignalReportTask.objects.filter(team_id=team_id, task_id__in=task_ids).values_list(+ "report_id", flat=True+ )+ )+ ids.update(+ str(report_id)+ for report_id in SignalReportAssignment.all_teams.filter(+ team_id=team_id, actor_task_id__in=task_ids+ ).values_list("report_id", flat=True)+ )+ return refresh_implementation_pr_cache(team_id=team_id, report_ids=ids)+++def schedule_implementation_pr_cache_refresh(+ *,+ team_id: int,+ report_ids: Iterable[Any] = (),+ pull_request_ids: Iterable[Any] = (),+ task_ids: Iterable[Any] = (),+) -> None:+ """Refresh after the current transaction commits, so the refresh reads the committed links.++ The caller passes only the ids it already holds. Resolving them to reports runs after the+ commit, inside the guard, so a failed lookup cannot fail the caller's write.+ """+ links = {+ "report_ids": sorted({str(value) for value in report_ids if value is not None}),+ "pull_request_ids": sorted({str(value) for value in pull_request_ids if value is not None}),+ "task_ids": sorted({str(value) for value in task_ids if value is not None}),+ }+ if not any(links.values()):+ return++ def refresh_after_commit() -> None:+ try:+ refresh_implementation_pr_cache_for_links(team_id=team_id, **links)+ except Exception:+ # The link write has committed, so its caller must not fail. The task retries the refresh.+ logger.warning("signals.implementation_pr_cache.refresh_failed", team_id=team_id, exc_info=True)+ from products.signals.backend.tasks import ( # noqa: PLC0415 — tasks imports this module+ refresh_report_implementation_pr_cache,+ )++ refresh_report_implementation_pr_cache.delay(team_id, **links)++ transaction.on_commit(refresh_after_commit, robust=True)+++def connect_implementation_pr_cache_receivers() -> None:+ # Save only on SignalReportAssignment: it is a read-only legacy table, and a delete receiver would+ # stop report deletes from fast-deleting its rows.+ post_save.connect(_refresh_for_report_link, sender=SignalReportAssignment, dispatch_uid="pr_cache_save_assignment")+ post_save.connect(_refresh_for_report_link, sender=SignalReportTask, dispatch_uid="pr_cache_save_report_task")+ post_delete.connect(_refresh_for_report_link, sender=SignalReportTask, dispatch_uid="pr_cache_delete_report_task")+ post_save.connect(_refresh_for_artefact, sender=SignalReportArtefact, dispatch_uid="pr_cache_save_artefact")+ post_delete.connect(_refresh_for_artefact, sender=SignalReportArtefact, dispatch_uid="pr_cache_delete_artefact")+ # Save only on SignalReportPullRequest and TaskRun: nothing deletes a pull request row outside a+ # team delete, and a delete receiver on the hot TaskRun table would stop cascades from+ # fast-deleting its rows. The daily reconcile repairs a report a manual delete leaves stale.+ post_save.connect(_refresh_for_pull_request, sender=SignalReportPullRequest, dispatch_uid="pr_cache_save_pr")+ connect_task_run_post_save(_refresh_for_task_run, dispatch_uid="signals_pr_cache_task_run")+++def _removes_the_report(origin: Any) -> bool:+ # Django passes the instance or queryset that `delete()` was called on as `origin`. A task+ # delete cascades into report links but leaves the reports, so only these origins skip the refresh.+ if origin is None:+ return False+ model = origin.model if isinstance(origin, QuerySet) else type(origin)+ return issubclass(model, _REPORT_REMOVING_ORIGINS)+++def _refresh_for_report_link(sender: type, instance: Any, **kwargs: Any) -> None:+ if _removes_the_report(kwargs.get("origin")):+ return+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, report_ids=[instance.report_id])+++def _refresh_for_artefact(sender: type, instance: SignalReportArtefact, **kwargs: Any) -> None:+ if instance.type not in _LINK_ARTEFACT_TYPES or _removes_the_report(kwargs.get("origin")):+ return+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, report_ids=[instance.report_id])+++def _refresh_for_pull_request(sender: type, instance: SignalReportPullRequest, **kwargs: Any) -> None:+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, pull_request_ids=[instance.id])+++def _refresh_for_task_run(sender: type, instance: Any, created: bool, **kwargs: Any) -> None:+ # TaskRun is a hot model, so this reads no rows. The filter reads the run's output and its+ # `state.ai_stage`. An output write can also remove a PR, so it always refreshes; a full save+ # is checked only when the run carries a PR, and the reconcile covers the rest.+ output = instance.output if isinstance(instance.output, dict) else {}+ carries_pr = bool(output.get("pr_url") or output.get("pr_urls"))+ update_fields = kwargs.get("update_fields")+ if created or update_fields is None:+ if not carries_pr:+ return+ elif "output" not in update_fields and not ("state" in update_fields and carries_pr):+ return+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, task_ids=[instance.task_id])+++def repair_implementation_pr_cache(+ *, team_id: int | None, batch_size: int, after: str | None = None+) -> Iterator[RepairedBatch]:+ """Recompute the cached flags of every report, one batch at a time, starting after report id `after`."""+ for batch in report_id_batches(team_id=team_id, batch_size=batch_size, after=after):+ by_team: dict[int, list[Any]] = defaultdict(list)+ for report_id, row_team_id in batch.rows:+ by_team[row_team_id].append(report_id)+ corrected = sum(+ refresh_implementation_pr_cache(team_id=row_team_id, report_ids=ids) for row_team_id, ids in by_team.items()+ )+ yield RepairedBatch(scanned=len(batch.rows), corrected=corrected, cursor=batch.cursor)diff --git a/products/signals/backend/management/AGENTS.md b/products/signals/backend/management/AGENTS.mdindex 2c4e093b8047..ba7cf69976c4 100644--- a/products/signals/backend/management/AGENTS.md+++ b/products/signals/backend/management/AGENTS.md@@ -211,6 +211,22 @@ uv run manage.py backfill_report_actionability --team-id 1 --batch-size 100 --afIdempotent. It recomputes each report from its artefacts and writes only the rows that disagree.+## Filling the cached implementation PR flags++`SignalReport.has_implementation_pr` and `has_active_implementation_pr` cache `implementation_pr_report_filter`, so the inbox filters on a column.+The receivers in `implementation_pr_cache.py` refresh them after every link write.+Run `backfill_report_inbox_state` once after the columns ship, and again as a repair after a bulk write that bypassed Django signals.++```bash+# Every team+uv run manage.py backfill_report_inbox_state++# One team, resuming after the report id an earlier run printed+uv run manage.py backfill_report_inbox_state --team-id 1 --after <report-id>+```++Idempotent. It writes only the rows that disagree with the live filter.+## Resuming operational scouts the setup flow pausedAn older self-driving setup flow switched off every scout the user did not pick, including operational scouts such as inbox validation.diff --git a/products/signals/backend/management/commands/backfill_report_inbox_state.py b/products/signals/backend/management/commands/backfill_report_inbox_state.pynew file mode 100644index 000000000000..f43bf57ef53a--- /dev/null+++ b/products/signals/backend/management/commands/backfill_report_inbox_state.py@@ -0,0 +1,26 @@+from argparse import ArgumentParser+from typing import cast++from django.core.management.base import BaseCommand++from products.signals.backend.implementation_pr_cache import repair_implementation_pr_cache+++class Command(BaseCommand):+ help = "Fill the cached implementation PR flags the inbox filters on. Safe to rerun."++ def add_arguments(self, parser: ArgumentParser) -> None:+ parser.add_argument("--team-id", type=int, help="Limit to one team. Omitted means every team.")+ parser.add_argument("--batch-size", type=int, default=500)+ parser.add_argument("--after", help="Resume after this report id, as printed by an earlier run.")++ def handle(self, *args: object, **options: object) -> None:+ scanned = corrected = 0+ for batch in repair_implementation_pr_cache(+ team_id=cast(int | None, options["team_id"]),+ batch_size=max(1, min(cast(int, options["batch_size"]), 5000)),+ after=cast(str | None, options["after"]),+ ):+ scanned += batch.scanned+ corrected += batch.corrected+ self.stdout.write(f"Scanned {scanned} reports, corrected {corrected}; resume after {batch.cursor}")diff --git a/products/signals/backend/migrations/0213_report_inbox_state_cache.py b/products/signals/backend/migrations/0213_report_inbox_state_cache.pynew file mode 100644index 000000000000..35f8ae01eda4--- /dev/null+++ b/products/signals/backend/migrations/0213_report_inbox_state_cache.py@@ -0,0 +1,22 @@+# Generated by Django 5.2.18 on 2026-10-27 19:38++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("signals", "0212_legacy_monitoring_check_provenance"),+ ]++ operations = [+ migrations.AddField(+ model_name="signalreport",+ name="has_active_implementation_pr",+ field=models.BooleanField(db_default=False, default=False),+ ),+ migrations.AddField(+ model_name="signalreport",+ name="has_implementation_pr",+ field=models.BooleanField(db_default=False, default=False),+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex be1ec2b1d67c..75a9222ee199 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0212_legacy_monitoring_check_provenance+0213_report_inbox_state_cachediff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 34050cfd5f71..4e389142ae60 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -489,6 +489,12 @@ class SignalReport(UUIDModel):latest_actionability = models.CharField(max_length=30, null=True, blank=True)latest_already_addressed = models.BooleanField(null=True, blank=True)+ # Cached from `implementation_pr_report_filter`, with and without `active_only`, so the inbox+ # filters on a column instead of evaluating five link subqueries for every report.+ # implementation_pr_cache.py keeps them current.+ has_implementation_pr = models.BooleanField(default=False, db_default=False)+ has_active_implementation_pr = models.BooleanField(default=False, db_default=False)+class Meta:indexes = [models.Index(fields=["team", "status", "promoted_at"]),diff --git a/products/signals/backend/report_actionability_repair.py b/products/signals/backend/report_actionability_repair.pyindex c0a04bb8f085..1e6a01ff8764 100644--- a/products/signals/backend/report_actionability_repair.py+++ b/products/signals/backend/report_actionability_repair.py@@ -1,4 +1,5 @@from collections.abc import Iterator+from uuid import UUIDfrom posthog.dataclasses import frozen@@ -12,16 +13,14 @@ class RepairedBatch:cursor: str-def repair_latest_actionability(- *, team_id: int | None, batch_size: int, after: str | None = None-) -> Iterator[RepairedBatch]:- """Recompute `SignalReport.latest_actionability` and `latest_already_addressed` from the- artefact log, one batch of reports at a time, starting after report id `after`.+@frozen+class ReportIdBatch:+ rows: list[tuple[UUID, int]]+ cursor: str- The receivers in receivers.py maintain both columns on every artefact write, so this is a- repair for rows that drifted, not a routine job. Safe to rerun: each report is recomputed- under its row lock and written only when it disagrees with its artefacts.- """++def report_id_batches(*, team_id: int | None, batch_size: int, after: str | None = None) -> Iterator[ReportIdBatch]:+ """Walk report ids in id order, as (id, team id) rows, starting after report id `after`."""reports = SignalReport.objects.all()if team_id is not None:reports = reports.filter(team_id=team_id)@@ -32,8 +31,22 @@ def repair_latest_actionability(if not rows:returnafter = str(rows[-1][0])+ yield ReportIdBatch(rows=rows, cursor=after)+++def repair_latest_actionability(+ *, team_id: int | None, batch_size: int, after: str | None = None+) -> Iterator[RepairedBatch]:+ """Recompute `SignalReport.latest_actionability` and `latest_already_addressed` from the+ artefact log, one batch of reports at a time, starting after report id `after`.++ The receivers in receivers.py maintain both columns on every artefact write, so this is a+ repair for rows that drifted, not a routine job. Safe to rerun: each report is recomputed+ under its row lock and written only when it disagrees with its artefacts.+ """+ for batch in report_id_batches(team_id=team_id, batch_size=batch_size, after=after):corrected = sum(SignalReport.refresh_latest_actionability(team_id=row_team_id, report_id=report_id)- for report_id, row_team_id in rows+ for report_id, row_team_id in batch.rows)- yield RepairedBatch(scanned=len(rows), corrected=corrected, cursor=after)+ yield RepairedBatch(scanned=len(batch.rows), corrected=corrected, cursor=batch.cursor)diff --git a/products/signals/backend/report_merge.py b/products/signals/backend/report_merge.pyindex a71fdb6c98b2..6d294a6ccd70 100644--- a/products/signals/backend/report_merge.py+++ b/products/signals/backend/report_merge.py@@ -35,6 +35,7 @@ from products.signals.backend.artefact_schemas import (ReportLink,)from products.signals.backend.enums import ReportLinkKind, ReportLinkWritePath+from products.signals.backend.implementation_pr_cache import schedule_implementation_pr_cache_refreshfrom products.signals.backend.models import (SignalReport,SignalReportArtefact,@@ -271,6 +272,9 @@ def _merge_source_into(# what the work log renders, is preserved..update(report_id=survivor.id, claim=None))+ # The queryset update fires no artefact receivers, and moved PR and task-run links change both+ # reports' cached implementation PR flags.+ schedule_implementation_pr_cache_refresh(team_id=source.team_id, report_ids=[source.id, survivor.id])_move_side_rows(source, survivor)diff --git a/products/signals/backend/tasks.py b/products/signals/backend/tasks.pyindex e4452c079d9f..a0455d571a33 100644--- a/products/signals/backend/tasks.py+++ b/products/signals/backend/tasks.py@@ -40,6 +40,7 @@ from products.signals.backend.implementation_dispatch_tasks import (sweep_implementation_dispatches as sweep_implementation_dispatches,)from products.signals.backend.implementation_pr import PrCloseReason, close_implementation_pr_for_report+from products.signals.backend.implementation_pr_cache import refresh_implementation_pr_cache_for_linksfrom products.signals.backend.inbox_push_notifications import notify_pull_request_ready, notify_report_visiblefrom products.signals.backend.inbox_summary import github_datetime, sync_pull_request_participantsfrom products.signals.backend.models import (@@ -93,6 +94,23 @@ from products.tasks.backend.facade.repo_activity import RepositoryCommitActivitylogger = structlog.get_logger(__name__)+@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=5)+@with_team_scope()+def refresh_report_implementation_pr_cache(+ team_id: int,+ report_ids: list[str] | None = None,+ pull_request_ids: list[str] | None = None,+ task_ids: list[str] | None = None,+) -> None:+ """Retry a cached PR flag refresh that failed after its link write committed."""+ refresh_implementation_pr_cache_for_links(+ team_id=team_id,+ report_ids=report_ids or [],+ pull_request_ids=pull_request_ids or [],+ task_ids=task_ids or [],+ )++@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=5)@with_team_scope()def write_pull_request_to_support_tickets(team_id: int, report_id: str, pr_url: str, task_id: str) -> None:diff --git a/products/signals/backend/test/test_backfill_report_inbox_state.py b/products/signals/backend/test/test_backfill_report_inbox_state.pynew file mode 100644index 000000000000..c207e3a88857--- /dev/null+++ b/products/signals/backend/test/test_backfill_report_inbox_state.py@@ -0,0 +1,47 @@+from posthog.test.base import BaseTest++from django.core.management import call_command++from products.signals.backend.models import SignalReport, SignalReportAssignment+++class TestBackfillReportInboxState(BaseTest):+ def _report(self) -> SignalReport:+ return SignalReport.objects.create(team=self.team, status=SignalReport.Status.READY, title="t", summary="s")++ def _flags(self, report: SignalReport) -> tuple[bool, bool]:+ row = SignalReport.objects.filter(id=report.id).values_list(+ "has_implementation_pr", "has_active_implementation_pr"+ )[0]+ return row[0], row[1]++ def test_corrects_implementation_pr_flags_that_disagree_with_the_links(self) -> None:+ open_pr = self._report()+ merged_pr = self._report()+ stale = self._report()+ for report, number, merged in ((open_pr, 1, False), (merged_pr, 2, True)):+ SignalReportAssignment.all_teams.create(+ team=self.team,+ report=report,+ pr_url=f"https://github.com/example/app/pull/{number}",+ repository="example/app",+ pr_number=number,+ pr_state="merged" if merged else "open",+ pr_merged=merged,+ )+ SignalReport.objects.filter(id__in=[open_pr.id, merged_pr.id]).update(+ has_implementation_pr=False, has_active_implementation_pr=False+ )+ SignalReport.objects.filter(id=stale.id).update(has_implementation_pr=True, has_active_implementation_pr=True)++ call_command(+ "backfill_report_inbox_state",+ "--team-id",+ str(self.team.id),+ "--batch-size",+ "2",+ )++ assert self._flags(open_pr) == (True, True)+ assert self._flags(merged_pr) == (True, False)+ assert self._flags(stale) == (False, False)diff --git a/products/signals/backend/test/test_implementation_pr_cache.py b/products/signals/backend/test/test_implementation_pr_cache.pynew file mode 100644index 000000000000..4549a0ff2347--- /dev/null+++ b/products/signals/backend/test/test_implementation_pr_cache.py@@ -0,0 +1,161 @@+import json+from collections.abc import Callable++from posthog.test.base import BaseTest+from unittest.mock import patch++from django.db import OperationalError++from parameterized import parameterized++from products.signals.backend.implementation_pr_cache import live_implementation_pr_flags+from products.signals.backend.models import (+ SignalReport,+ SignalReportArtefact,+ SignalReportAssignment,+ SignalReportPullRequest,+ SignalReportTask,+)+from products.tasks.backend.models import Task, TaskRun++PR_URL = "https://github.com/example/app/pull/7"+++class TestImplementationPrCache(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.report = SignalReport.objects.create(+ team=self.team, status=SignalReport.Status.READY, title="t", summary="s"+ )++ def _cached_and_live(self) -> tuple[tuple[bool, bool], tuple[bool, bool]]:+ live = live_implementation_pr_flags(self.team.id)+ row = (+ SignalReport.objects.filter(id=self.report.id)+ .annotate(live_has_implementation_pr=live.has_pr, live_has_active_implementation_pr=live.has_active_pr)+ .values(+ "has_implementation_pr",+ "has_active_implementation_pr",+ "live_has_implementation_pr",+ "live_has_active_implementation_pr",+ )[0]+ )+ return (+ (row["has_implementation_pr"], row["has_active_implementation_pr"]),+ (row["live_has_implementation_pr"], row["live_has_active_implementation_pr"]),+ )++ def _task(self, *, with_pr: bool) -> tuple[Task, TaskRun]:+ task = Task.objects.create(team=self.team, title="Implementation", description="", origin_product="signals")+ run = TaskRun.objects.create(team=self.team, task=task, status=TaskRun.Status.COMPLETED)+ if with_pr:+ # A queryset update, so the run's own receiver is not what links it here.+ TaskRun.objects.filter(id=run.id).update(output={"pr_url": PR_URL})+ return task, run++ def _task_run_artefact(self, task: Task, run: TaskRun) -> None:+ SignalReportArtefact.objects.create(+ team=self.team,+ report=self.report,+ type=SignalReportArtefact.ArtefactType.TASK_RUN,+ task_id=task.id,+ content=json.dumps(+ {"task_id": str(task.id), "run_id": str(run.id), "product": "signals", "type": "implementation"}+ ),+ )++ def _link_assignment(self) -> None:+ SignalReportAssignment.all_teams.create(+ team=self.team, report=self.report, pr_url=PR_URL, repository="example/app", pr_number=7, pr_state="open"+ )++ def _link_pull_request(self) -> None:+ pull_request = SignalReportPullRequest.objects.create(+ team=self.team, repository="example/app", number=7, url=PR_URL, state="open"+ )+ SignalReportArtefact.objects.create(+ team=self.team,+ report=self.report,+ type=SignalReportArtefact.ArtefactType.PULL_REQUEST,+ pull_request=pull_request,+ content="{}",+ )++ def _link_task_run_artefact(self) -> None:+ self._task_run_artefact(*self._task(with_pr=True))++ def _link_task_run_output(self) -> None:+ task, run = self._task(with_pr=False)+ self._task_run_artefact(task, run)+ run.output = {"pr_url": PR_URL}+ # The assignment sync on the same save queues tracker linking, which retries without a provider.+ with patch("products.signals.backend.tasks.link_report_tracker_issues.delay"):+ run.save(update_fields=["output"])++ def _link_legacy_report_task(self) -> None:+ task, _ = self._task(with_pr=True)+ SignalReportTask.objects.create(team=self.team, report=self.report, task=task, relationship="implementation")++ @parameterized.expand(+ [+ ("assignment", "_link_assignment"),+ ("pull_request_artefact", "_link_pull_request"),+ ("task_run_artefact", "_link_task_run_artefact"),+ ("task_run_output_written_after_the_link", "_link_task_run_output"),+ ("legacy_report_task", "_link_legacy_report_task"),+ ]+ )+ def test_each_link_source_keeps_the_cached_flags_equal_to_the_live_filter(self, _name: str, link: str) -> None:+ write: Callable[[], None] = getattr(self, link)+ with self.captureOnCommitCallbacks(execute=True):+ write()++ cached, live = self._cached_and_live()+ assert live == (True, True)+ assert cached == live++ def test_a_merged_pull_request_stops_counting_as_active(self) -> None:+ with self.captureOnCommitCallbacks(execute=True):+ self._link_pull_request()+ pull_request = SignalReportPullRequest.objects.get(team=self.team, url=PR_URL)++ with self.captureOnCommitCallbacks(execute=True):+ pull_request.state = "merged"+ pull_request.save()++ cached, live = self._cached_and_live()+ assert live == (True, False)+ assert cached == live++ @parameterized.expand([("output_cleared", "clear_output"), ("task_deleted", "delete_task")])+ def test_removing_the_only_task_pr_clears_the_cached_flags(self, _name: str, removal: str) -> None:+ task, run = self._task(with_pr=True)+ SignalReportTask.objects.create(team=self.team, report=self.report, task=task, relationship="implementation")+ self._task_run_artefact(task, run)+ SignalReport.objects.filter(id=self.report.id).update(+ has_implementation_pr=True, has_active_implementation_pr=True+ )++ with self.captureOnCommitCallbacks(execute=True):+ if removal == "clear_output":+ run.output = None+ run.save(update_fields=["output"])+ else:+ Task.objects.filter(id=task.id).delete()++ cached, live = self._cached_and_live()+ assert live == (False, False)+ assert cached == live++ def test_a_failed_refresh_is_retried_by_a_task(self) -> None:+ with (+ patch(+ "products.signals.backend.implementation_pr_cache.refresh_implementation_pr_cache",+ side_effect=OperationalError("connection reset"),+ ),+ patch("products.signals.backend.tasks.refresh_report_implementation_pr_cache.delay") as retry,+ self.captureOnCommitCallbacks(execute=True),+ ):+ self._link_assignment()++ retry.assert_called_once_with(self.team.id, report_ids=[str(self.report.id)], pull_request_ids=[], task_ids=[])