Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): cache implementation PR state on reports (#124388)

Dylan Martin09d34bf4b38d
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txt
index bb0609ecf09b..d776d5d1b54c 100644
--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt
+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt
@@ -116,6 +116,8 @@ post_delete:products.feature_flags.backend.local_evaluation.feature_flag_changed
post_delete:products.managed_warehouse.backend.models._observe_duckgres_server_delete
post_delete:products.replay_vision.backend.media_expiry.expire_media_asset
post_delete:products.review_hog.backend.ownership._invalidate_owned_repositories_on_delete
+post_delete:products.signals.backend.implementation_pr_cache._refresh_for_artefact
+post_delete:products.signals.backend.implementation_pr_cache._refresh_for_report_link
post_delete:products.signals.backend.receivers.mark_scout_suggestions_stale_on_fleet_change
post_delete:products.signals.backend.receivers.reconcile_report_embedding_on_verdict_deleted
post_delete:products.signals.backend.receivers.sync_report_latest_actionability_on_delete
@@ -240,6 +242,10 @@ post_save:products.metrics.backend.dashboard_import.receivers.schedule_import_fi
post_save:products.posthog_ai.backend.receivers.mark_conversation_sandbox_on_first_run
post_save:products.review_hog.backend.ownership._invalidate_owned_repositories_on_save
post_save:products.review_hog.backend.receivers.handle_task_run_saved
+post_save:products.signals.backend.implementation_pr_cache._refresh_for_artefact
+post_save:products.signals.backend.implementation_pr_cache._refresh_for_pull_request
+post_save:products.signals.backend.implementation_pr_cache._refresh_for_report_link
+post_save:products.signals.backend.implementation_pr_cache._refresh_for_task_run
post_save:products.signals.backend.receivers.arm_pending_checks_when_report_monitoring
post_save:products.signals.backend.receivers.capture_status_change_analytics
post_save:products.signals.backend.receivers.close_pr_when_report_dismissed
diff --git a/products/signals/backend/apps.py b/products/signals/backend/apps.py
index 1b18a761959c..57fc582ca7ad 100644
--- a/products/signals/backend/apps.py
+++ b/products/signals/backend/apps.py
@@ -20,10 +20,12 @@ class SignalsConfig(AppConfig):
# receivers: post_save receiver that closes a report's implementation PR on suppression/snooze.
from . import (
activity_logging, # noqa: F401
+ implementation_pr_cache,
receivers, # noqa: F401
)
receivers.connect_task_run_assignment_sync()
+ implementation_pr_cache.connect_implementation_pr_cache_receivers()
self._register_task_spend_updates()
self._register_signal_emission_gate()
self._register_scout_skill_archive_guard()
diff --git a/products/signals/backend/implementation_pr.py b/products/signals/backend/implementation_pr.py
index d7c888b06028..7eed6fd49674 100644
--- a/products/signals/backend/implementation_pr.py
+++ b/products/signals/backend/implementation_pr.py
@@ -1,14 +1,14 @@
"""Resolve implementation PR URLs linked to signal reports."""
import re
-from collections.abc import Callable
+from collections.abc import Callable, Sequence
from dataclasses import replace
from datetime import datetime
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any, Literal, TypeVar
from uuid import NAMESPACE_URL, uuid5
from django.contrib.postgres.expressions import ArraySubquery
-from django.db.models import F, Lookup, Q
+from django.db.models import F, Lookup, Q, QuerySet
import structlog
@@ -38,6 +38,8 @@ from products.tasks.backend.facade import api as tasks_facade
logger = structlog.get_logger(__name__)
+_LinkModel = TypeVar("_LinkModel", SignalReportArtefact, SignalReportAssignment, SignalReportTask)
+
if TYPE_CHECKING:
from django.db.backends.base.base import BaseDatabaseWrapper
from django.db.models.sql.compiler import SQLCompiler
@@ -67,34 +69,69 @@ class _EqualsAny(Lookup):
return f"{lhs} = ANY({rhs})", (*lhs_params, *rhs_params)
-def implementation_pr_report_filter(*, team_id: int, active_only: bool = False) -> Q:
+def implementation_pr_report_filter(
+ *, team_id: int, active_only: bool = False, report_ids: Sequence[Any] | None = None
+) -> Q:
+ """Matches the reports with an implementation PR.
+
+ `report_ids` narrows every link subquery to those reports, so a caller that asks about a few
+ reports does not scan the team's links and PR-bearing task runs. The answer for those reports
+ is the same either way.
+ """
+
# Every branch is a team-scoped subquery on the report id. A join through `assignment` or
# `pull_request` lets the planner scan every team's rows before it applies the team scope.
- assignments = SignalReportAssignment.all_teams.filter(team_id=team_id, pr_url__regex=_PULL_REQUEST_URL_PATTERN)
+ def for_reports(queryset: QuerySet[_LinkModel]) -> QuerySet[_LinkModel]:
+ return queryset if report_ids is None else queryset.filter(report_id__in=report_ids)
+
+ assignments = for_reports(
+ SignalReportAssignment.all_teams.filter(team_id=team_id, pr_url__regex=_PULL_REQUEST_URL_PATTERN)
+ )
pull_requests = SignalReportPullRequest.objects.for_team(team_id).filter(url__regex=_PULL_REQUEST_URL_PATTERN)
- task_ids = tasks_facade.task_ids_with_pr_url_subquery(
- team_id,
+ task_conditions = [
pr_bearing_task_run_filter(),
Q(output__pr_url__regex=_PULL_REQUEST_URL_PATTERN) | Q(output__pr_urls__regex=_PULL_REQUEST_URL_ARRAY_PATTERN),
- )
+ ]
+ if report_ids is not None:
+ pull_requests = pull_requests.filter(
+ id__in=SignalReportArtefact.objects.filter(
+ team_id=team_id, type=SignalReportArtefact.ArtefactType.PULL_REQUEST, report_id__in=report_ids
+ ).values("pull_request_id")
+ )
+ task_conditions.append(
+ Q(task_id__in=for_reports(SignalReportArtefact.objects.filter(team_id=team_id)).values("task_id"))
+ | Q(task_id__in=for_reports(SignalReportTask.objects.filter(team_id=team_id)).values("task_id"))
+ | Q(
+ task_id__in=for_reports(SignalReportAssignment.all_teams.filter(team_id=team_id)).values(
+ "actor_task_id"
+ )
+ )
+ )
+ task_ids = tasks_facade.task_ids_with_pr_url_subquery(team_id, *task_conditions)
task_run_artefacts = (
- SignalReportArtefact.objects.filter(
- team_id=team_id,
- type=SignalReportArtefact.ArtefactType.TASK_RUN,
- task_id__in=task_ids,
- content__regex=rf'"product"\s*:\s*"{SIGNALS_PRODUCT}"',
+ for_reports(
+ SignalReportArtefact.objects.filter(
+ team_id=team_id,
+ type=SignalReportArtefact.ArtefactType.TASK_RUN,
+ task_id__in=task_ids,
+ content__regex=rf'"product"\s*:\s*"{SIGNALS_PRODUCT}"',
+ )
)
.exclude(content__regex=rf'"type"\s*:\s*"({"|".join(sorted(NON_PR_BEARING_TASK_RUN_TYPES))})"')
.values("report_id")
)
- legacy_tasks = SignalReportTask.objects.filter(
- team_id=team_id,
- task_id__in=task_ids,
- relationship__in=_PR_BEARING_LEGACY_TASK_RELATIONSHIPS,
+ legacy_tasks = for_reports(
+ SignalReportTask.objects.filter(
+ team_id=team_id,
+ task_id__in=task_ids,
+ relationship__in=_PR_BEARING_LEGACY_TASK_RELATIONSHIPS,
+ )
).values("report_id")
- assignment_tasks = SignalReportAssignment.all_teams.filter(
- team_id=team_id,
- actor_task_id__in=task_ids,
+ assignment_tasks = for_reports(
+ SignalReportAssignment.all_teams.filter(
+ team_id=team_id,
+ actor_task_id__in=task_ids,
+ )
).values("report_id")
if active_only:
@@ -110,10 +147,12 @@ def implementation_pr_report_filter(*, team_id: int, active_only: bool = False)
# `pull_request_id IN (subquery)` keeps the cross-team merge join. Only the array form makes
# Postgres probe `signals_artefact_pr_report_idx` with this team's PR ids.
- pull_request_links = SignalReportArtefact.objects.filter(
- _EqualsAny(F("pull_request_id"), ArraySubquery(pull_requests.values("id"))),
- team_id=team_id,
- type=SignalReportArtefact.ArtefactType.PULL_REQUEST,
+ pull_request_links = for_reports(
+ SignalReportArtefact.objects.filter(
+ _EqualsAny(F("pull_request_id"), ArraySubquery(pull_requests.values("id"))),
+ team_id=team_id,
+ type=SignalReportArtefact.ArtefactType.PULL_REQUEST,
+ )
).values("report_id")
return (
diff --git a/products/signals/backend/implementation_pr_cache.py b/products/signals/backend/implementation_pr_cache.py
new file mode 100644
index 000000000000..dd158538f0fd
--- /dev/null
+++ b/products/signals/backend/implementation_pr_cache.py
@@ -0,0 +1,255 @@
+"""Keep `SignalReport.has_implementation_pr` and `has_active_implementation_pr` equal to the link tables.
+
+`implementation_pr_report_filter` is the source of truth. It reads five link sources, so the inbox
+cannot afford to evaluate it for every report on each list and count. These helpers store its
+answer on the report row instead. Receivers call `schedule_implementation_pr_cache_refresh` after a
+link write, and the daily reconcile corrects rows that a write path outside the receivers left stale.
+"""
+
+from collections import defaultdict
+from collections.abc import Iterable, Iterator, Sequence
+from typing import Any
+
+from django.db import transaction
+from django.db.models import BooleanField, ExpressionWrapper, QuerySet
+from django.db.models.signals import post_delete, post_save
+
+import structlog
+
+from posthog.dataclasses import frozen
+from posthog.models.organization import Organization
+from posthog.models.project import Project
+from posthog.models.team import Team
+
+from products.signals.backend.models import (
+ SignalReport,
+ SignalReportArtefact,
+ SignalReportAssignment,
+ SignalReportPullRequest,
+ SignalReportTask,
+)
+from products.signals.backend.report_actionability_repair import RepairedBatch, report_id_batches
+from products.tasks.backend.facade.task_run_signals import connect_task_run_post_save
+
+logger = structlog.get_logger(__name__)
+
+_LINK_ARTEFACT_TYPES = frozenset(
+ {SignalReportArtefact.ArtefactType.PULL_REQUEST, SignalReportArtefact.ArtefactType.TASK_RUN}
+)
+# A delete that starts at one of these removes the linked reports too, so their links need no refresh.
+_REPORT_REMOVING_ORIGINS = (SignalReport, Team, Project, Organization)
+
+
+@frozen
+class LiveImplementationPrFlags:
+ has_pr: ExpressionWrapper
+ has_active_pr: ExpressionWrapper
+
+
+def live_implementation_pr_flags(team_id: int, report_ids: Sequence[Any] | None = None) -> LiveImplementationPrFlags:
+ """The source-of-truth filter as two per-report annotations: any PR, and an active PR."""
+ from products.signals.backend.implementation_pr import ( # noqa: PLC0415 — keeps the tasks facade off the Django setup path, which imports this module for its receivers
+ implementation_pr_report_filter,
+ )
+
+ return LiveImplementationPrFlags(
+ has_pr=ExpressionWrapper(
+ implementation_pr_report_filter(team_id=team_id, report_ids=report_ids), output_field=BooleanField()
+ ),
+ has_active_pr=ExpressionWrapper(
+ implementation_pr_report_filter(team_id=team_id, active_only=True, report_ids=report_ids),
+ output_field=BooleanField(),
+ ),
+ )
+
+
+def refresh_implementation_pr_cache(*, team_id: int, report_ids: Iterable[Any]) -> int:
+ """Recompute the cached flags of these reports and store the ones that changed.
+
+ Returns how many reports changed. The rows stay locked from the read to the write, so a refresh
+ that read the links before a concurrent link write committed cannot overwrite the newer answer.
+ """
+ ids = sorted({str(report_id) for report_id in report_ids})
+ if not ids:
+ return 0
+ with transaction.atomic():
+ # FOR NO KEY UPDATE, for the reason `refresh_latest_actionability` gives: link inserts hold
+ # KEY SHARE on the report through their foreign key.
+ locked = list(
+ SignalReport.objects.select_for_update(no_key=True)
+ .filter(team_id=team_id, id__in=ids)
+ .order_by("id")
+ .values_list("id", flat=True)
+ )
+ if not locked:
+ return 0
+ live = live_implementation_pr_flags(team_id, report_ids=locked)
+ rows = (
+ SignalReport.objects.filter(team_id=team_id, id__in=locked)
+ .annotate(live_has_implementation_pr=live.has_pr, live_has_active_implementation_pr=live.has_active_pr)
+ .values_list(
+ "id",
+ "has_implementation_pr",
+ "has_active_implementation_pr",
+ "live_has_implementation_pr",
+ "live_has_active_implementation_pr",
+ )
+ )
+ changed: dict[tuple[bool, bool], list[Any]] = defaultdict(list)
+ for report_id, cached_has, cached_active, has_pr, has_active_pr in rows:
+ if (cached_has, cached_active) != (has_pr, has_active_pr):
+ changed[(has_pr, has_active_pr)].append(report_id)
+ for (has_pr, has_active_pr), changed_ids in changed.items():
+ # `update()`, not `save()`: a cache refresh must not bump `updated_at`, which the inbox
+ # sorts on, or fire the report's save receivers.
+ SignalReport.objects.filter(team_id=team_id, id__in=changed_ids).update(
+ has_implementation_pr=has_pr, has_active_implementation_pr=has_active_pr
+ )
+ return sum(len(changed_ids) for changed_ids in changed.values())
+
+
+def refresh_implementation_pr_cache_for_links(
+ *,
+ team_id: int,
+ report_ids: Iterable[Any] = (),
+ pull_request_ids: Iterable[Any] = (),
+ task_ids: Iterable[Any] = (),
+) -> int:
+ """Refresh the reports these links reach: the reports themselves, the reports a pull request
+ links to, and the reports a task links to."""
+ ids = {str(report_id) for report_id in report_ids}
+ pull_request_ids = list(pull_request_ids)
+ task_ids = list(task_ids)
+ if pull_request_ids:
+ ids.update(
+ str(report_id)
+ for report_id in SignalReportArtefact.objects.filter(
+ team_id=team_id,
+ type=SignalReportArtefact.ArtefactType.PULL_REQUEST,
+ pull_request_id__in=pull_request_ids,
+ ).values_list("report_id", flat=True)
+ )
+ if task_ids:
+ ids.update(
+ str(report_id)
+ for report_id in SignalReportArtefact.objects.filter(
+ team_id=team_id, type=SignalReportArtefact.ArtefactType.TASK_RUN, task_id__in=task_ids
+ ).values_list("report_id", flat=True)
+ )
+ ids.update(
+ str(report_id)
+ for report_id in SignalReportTask.objects.filter(team_id=team_id, task_id__in=task_ids).values_list(
+ "report_id", flat=True
+ )
+ )
+ ids.update(
+ str(report_id)
+ for report_id in SignalReportAssignment.all_teams.filter(
+ team_id=team_id, actor_task_id__in=task_ids
+ ).values_list("report_id", flat=True)
+ )
+ return refresh_implementation_pr_cache(team_id=team_id, report_ids=ids)
+
+
+def schedule_implementation_pr_cache_refresh(
+ *,
+ team_id: int,
+ report_ids: Iterable[Any] = (),
+ pull_request_ids: Iterable[Any] = (),
+ task_ids: Iterable[Any] = (),
+) -> None:
+ """Refresh after the current transaction commits, so the refresh reads the committed links.
+
+ The caller passes only the ids it already holds. Resolving them to reports runs after the
+ commit, inside the guard, so a failed lookup cannot fail the caller's write.
+ """
+ links = {
+ "report_ids": sorted({str(value) for value in report_ids if value is not None}),
+ "pull_request_ids": sorted({str(value) for value in pull_request_ids if value is not None}),
+ "task_ids": sorted({str(value) for value in task_ids if value is not None}),
+ }
+ if not any(links.values()):
+ return
+
+ def refresh_after_commit() -> None:
+ try:
+ refresh_implementation_pr_cache_for_links(team_id=team_id, **links)
+ except Exception:
+ # The link write has committed, so its caller must not fail. The task retries the refresh.
+ logger.warning("signals.implementation_pr_cache.refresh_failed", team_id=team_id, exc_info=True)
+ from products.signals.backend.tasks import ( # noqa: PLC0415 — tasks imports this module
+ refresh_report_implementation_pr_cache,
+ )
+
+ refresh_report_implementation_pr_cache.delay(team_id, **links)
+
+ transaction.on_commit(refresh_after_commit, robust=True)
+
+
+def connect_implementation_pr_cache_receivers() -> None:
+ # Save only on SignalReportAssignment: it is a read-only legacy table, and a delete receiver would
+ # stop report deletes from fast-deleting its rows.
+ post_save.connect(_refresh_for_report_link, sender=SignalReportAssignment, dispatch_uid="pr_cache_save_assignment")
+ post_save.connect(_refresh_for_report_link, sender=SignalReportTask, dispatch_uid="pr_cache_save_report_task")
+ post_delete.connect(_refresh_for_report_link, sender=SignalReportTask, dispatch_uid="pr_cache_delete_report_task")
+ post_save.connect(_refresh_for_artefact, sender=SignalReportArtefact, dispatch_uid="pr_cache_save_artefact")
+ post_delete.connect(_refresh_for_artefact, sender=SignalReportArtefact, dispatch_uid="pr_cache_delete_artefact")
+ # Save only on SignalReportPullRequest and TaskRun: nothing deletes a pull request row outside a
+ # team delete, and a delete receiver on the hot TaskRun table would stop cascades from
+ # fast-deleting its rows. The daily reconcile repairs a report a manual delete leaves stale.
+ post_save.connect(_refresh_for_pull_request, sender=SignalReportPullRequest, dispatch_uid="pr_cache_save_pr")
+ connect_task_run_post_save(_refresh_for_task_run, dispatch_uid="signals_pr_cache_task_run")
+
+
+def _removes_the_report(origin: Any) -> bool:
+ # Django passes the instance or queryset that `delete()` was called on as `origin`. A task
+ # delete cascades into report links but leaves the reports, so only these origins skip the refresh.
+ if origin is None:
+ return False
+ model = origin.model if isinstance(origin, QuerySet) else type(origin)
+ return issubclass(model, _REPORT_REMOVING_ORIGINS)
+
+
+def _refresh_for_report_link(sender: type, instance: Any, **kwargs: Any) -> None:
+ if _removes_the_report(kwargs.get("origin")):
+ return
+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, report_ids=[instance.report_id])
+
+
+def _refresh_for_artefact(sender: type, instance: SignalReportArtefact, **kwargs: Any) -> None:
+ if instance.type not in _LINK_ARTEFACT_TYPES or _removes_the_report(kwargs.get("origin")):
+ return
+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, report_ids=[instance.report_id])
+
+
+def _refresh_for_pull_request(sender: type, instance: SignalReportPullRequest, **kwargs: Any) -> None:
+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, pull_request_ids=[instance.id])
+
+
+def _refresh_for_task_run(sender: type, instance: Any, created: bool, **kwargs: Any) -> None:
+ # TaskRun is a hot model, so this reads no rows. The filter reads the run's output and its
+ # `state.ai_stage`. An output write can also remove a PR, so it always refreshes; a full save
+ # is checked only when the run carries a PR, and the reconcile covers the rest.
+ output = instance.output if isinstance(instance.output, dict) else {}
+ carries_pr = bool(output.get("pr_url") or output.get("pr_urls"))
+ update_fields = kwargs.get("update_fields")
+ if created or update_fields is None:
+ if not carries_pr:
+ return
+ elif "output" not in update_fields and not ("state" in update_fields and carries_pr):
+ return
+ schedule_implementation_pr_cache_refresh(team_id=instance.team_id, task_ids=[instance.task_id])
+
+
+def repair_implementation_pr_cache(
+ *, team_id: int | None, batch_size: int, after: str | None = None
+) -> Iterator[RepairedBatch]:
+ """Recompute the cached flags of every report, one batch at a time, starting after report id `after`."""
+ for batch in report_id_batches(team_id=team_id, batch_size=batch_size, after=after):
+ by_team: dict[int, list[Any]] = defaultdict(list)
+ for report_id, row_team_id in batch.rows:
+ by_team[row_team_id].append(report_id)
+ corrected = sum(
+ refresh_implementation_pr_cache(team_id=row_team_id, report_ids=ids) for row_team_id, ids in by_team.items()
+ )
+ yield RepairedBatch(scanned=len(batch.rows), corrected=corrected, cursor=batch.cursor)
diff --git a/products/signals/backend/management/AGENTS.md b/products/signals/backend/management/AGENTS.md
index 2c4e093b8047..ba7cf69976c4 100644
--- a/products/signals/backend/management/AGENTS.md
+++ b/products/signals/backend/management/AGENTS.md
@@ -211,6 +211,22 @@ uv run manage.py backfill_report_actionability --team-id 1 --batch-size 100 --af
Idempotent. It recomputes each report from its artefacts and writes only the rows that disagree.
+## Filling the cached implementation PR flags
+
+`SignalReport.has_implementation_pr` and `has_active_implementation_pr` cache `implementation_pr_report_filter`, so the inbox filters on a column.
+The receivers in `implementation_pr_cache.py` refresh them after every link write.
+Run `backfill_report_inbox_state` once after the columns ship, and again as a repair after a bulk write that bypassed Django signals.
+
+```bash
+# Every team
+uv run manage.py backfill_report_inbox_state
+
+# One team, resuming after the report id an earlier run printed
+uv run manage.py backfill_report_inbox_state --team-id 1 --after <report-id>
+```
+
+Idempotent. It writes only the rows that disagree with the live filter.
+
## Resuming operational scouts the setup flow paused
An older self-driving setup flow switched off every scout the user did not pick, including operational scouts such as inbox validation.
diff --git a/products/signals/backend/management/commands/backfill_report_inbox_state.py b/products/signals/backend/management/commands/backfill_report_inbox_state.py
new file mode 100644
index 000000000000..f43bf57ef53a
--- /dev/null
+++ b/products/signals/backend/management/commands/backfill_report_inbox_state.py
@@ -0,0 +1,26 @@
+from argparse import ArgumentParser
+from typing import cast
+
+from django.core.management.base import BaseCommand
+
+from products.signals.backend.implementation_pr_cache import repair_implementation_pr_cache
+
+
+class Command(BaseCommand):
+ help = "Fill the cached implementation PR flags the inbox filters on. Safe to rerun."
+
+ def add_arguments(self, parser: ArgumentParser) -> None:
+ parser.add_argument("--team-id", type=int, help="Limit to one team. Omitted means every team.")
+ parser.add_argument("--batch-size", type=int, default=500)
+ parser.add_argument("--after", help="Resume after this report id, as printed by an earlier run.")
+
+ def handle(self, *args: object, **options: object) -> None:
+ scanned = corrected = 0
+ for batch in repair_implementation_pr_cache(
+ team_id=cast(int | None, options["team_id"]),
+ batch_size=max(1, min(cast(int, options["batch_size"]), 5000)),
+ after=cast(str | None, options["after"]),
+ ):
+ scanned += batch.scanned
+ corrected += batch.corrected
+ self.stdout.write(f"Scanned {scanned} reports, corrected {corrected}; resume after {batch.cursor}")
diff --git a/products/signals/backend/migrations/0213_report_inbox_state_cache.py b/products/signals/backend/migrations/0213_report_inbox_state_cache.py
new file mode 100644
index 000000000000..35f8ae01eda4
--- /dev/null
+++ b/products/signals/backend/migrations/0213_report_inbox_state_cache.py
@@ -0,0 +1,22 @@
+# Generated by Django 5.2.18 on 2026-10-27 19:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0212_legacy_monitoring_check_provenance"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="signalreport",
+ name="has_active_implementation_pr",
+ field=models.BooleanField(db_default=False, default=False),
+ ),
+ migrations.AddField(
+ model_name="signalreport",
+ name="has_implementation_pr",
+ field=models.BooleanField(db_default=False, default=False),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index be1ec2b1d67c..75a9222ee199 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0212_legacy_monitoring_check_provenance
+0213_report_inbox_state_cache
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 34050cfd5f71..4e389142ae60 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -489,6 +489,12 @@ class SignalReport(UUIDModel):
latest_actionability = models.CharField(max_length=30, null=True, blank=True)
latest_already_addressed = models.BooleanField(null=True, blank=True)
+ # Cached from `implementation_pr_report_filter`, with and without `active_only`, so the inbox
+ # filters on a column instead of evaluating five link subqueries for every report.
+ # implementation_pr_cache.py keeps them current.
+ has_implementation_pr = models.BooleanField(default=False, db_default=False)
+ has_active_implementation_pr = models.BooleanField(default=False, db_default=False)
+
class Meta:
indexes = [
models.Index(fields=["team", "status", "promoted_at"]),
diff --git a/products/signals/backend/report_actionability_repair.py b/products/signals/backend/report_actionability_repair.py
index c0a04bb8f085..1e6a01ff8764 100644
--- a/products/signals/backend/report_actionability_repair.py
+++ b/products/signals/backend/report_actionability_repair.py
@@ -1,4 +1,5 @@
from collections.abc import Iterator
+from uuid import UUID
from posthog.dataclasses import frozen
@@ -12,16 +13,14 @@ class RepairedBatch:
cursor: str
-def repair_latest_actionability(
- *, team_id: int | None, batch_size: int, after: str | None = None
-) -> Iterator[RepairedBatch]:
- """Recompute `SignalReport.latest_actionability` and `latest_already_addressed` from the
- artefact log, one batch of reports at a time, starting after report id `after`.
+@frozen
+class ReportIdBatch:
+ rows: list[tuple[UUID, int]]
+ cursor: str
- The receivers in receivers.py maintain both columns on every artefact write, so this is a
- repair for rows that drifted, not a routine job. Safe to rerun: each report is recomputed
- under its row lock and written only when it disagrees with its artefacts.
- """
+
+def report_id_batches(*, team_id: int | None, batch_size: int, after: str | None = None) -> Iterator[ReportIdBatch]:
+ """Walk report ids in id order, as (id, team id) rows, starting after report id `after`."""
reports = SignalReport.objects.all()
if team_id is not None:
reports = reports.filter(team_id=team_id)
@@ -32,8 +31,22 @@ def repair_latest_actionability(
if not rows:
return
after = str(rows[-1][0])
+ yield ReportIdBatch(rows=rows, cursor=after)
+
+
+def repair_latest_actionability(
+ *, team_id: int | None, batch_size: int, after: str | None = None
+) -> Iterator[RepairedBatch]:
+ """Recompute `SignalReport.latest_actionability` and `latest_already_addressed` from the
+ artefact log, one batch of reports at a time, starting after report id `after`.
+
+ The receivers in receivers.py maintain both columns on every artefact write, so this is a
+ repair for rows that drifted, not a routine job. Safe to rerun: each report is recomputed
+ under its row lock and written only when it disagrees with its artefacts.
+ """
+ for batch in report_id_batches(team_id=team_id, batch_size=batch_size, after=after):
corrected = sum(
SignalReport.refresh_latest_actionability(team_id=row_team_id, report_id=report_id)
- for report_id, row_team_id in rows
+ for report_id, row_team_id in batch.rows
)
- yield RepairedBatch(scanned=len(rows), corrected=corrected, cursor=after)
+ yield RepairedBatch(scanned=len(batch.rows), corrected=corrected, cursor=batch.cursor)
diff --git a/products/signals/backend/report_merge.py b/products/signals/backend/report_merge.py
index a71fdb6c98b2..6d294a6ccd70 100644
--- a/products/signals/backend/report_merge.py
+++ b/products/signals/backend/report_merge.py
@@ -35,6 +35,7 @@ from products.signals.backend.artefact_schemas import (
ReportLink,
)
from products.signals.backend.enums import ReportLinkKind, ReportLinkWritePath
+from products.signals.backend.implementation_pr_cache import schedule_implementation_pr_cache_refresh
from products.signals.backend.models import (
SignalReport,
SignalReportArtefact,
@@ -271,6 +272,9 @@ def _merge_source_into(
# what the work log renders, is preserved.
.update(report_id=survivor.id, claim=None)
)
+ # The queryset update fires no artefact receivers, and moved PR and task-run links change both
+ # reports' cached implementation PR flags.
+ schedule_implementation_pr_cache_refresh(team_id=source.team_id, report_ids=[source.id, survivor.id])
_move_side_rows(source, survivor)
diff --git a/products/signals/backend/tasks.py b/products/signals/backend/tasks.py
index e4452c079d9f..a0455d571a33 100644
--- a/products/signals/backend/tasks.py
+++ b/products/signals/backend/tasks.py
@@ -40,6 +40,7 @@ from products.signals.backend.implementation_dispatch_tasks import (
sweep_implementation_dispatches as sweep_implementation_dispatches,
)
from products.signals.backend.implementation_pr import PrCloseReason, close_implementation_pr_for_report
+from products.signals.backend.implementation_pr_cache import refresh_implementation_pr_cache_for_links
from products.signals.backend.inbox_push_notifications import notify_pull_request_ready, notify_report_visible
from products.signals.backend.inbox_summary import github_datetime, sync_pull_request_participants
from products.signals.backend.models import (
@@ -93,6 +94,23 @@ from products.tasks.backend.facade.repo_activity import RepositoryCommitActivity
logger = structlog.get_logger(__name__)
+@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=5)
+@with_team_scope()
+def refresh_report_implementation_pr_cache(
+ team_id: int,
+ report_ids: list[str] | None = None,
+ pull_request_ids: list[str] | None = None,
+ task_ids: list[str] | None = None,
+) -> None:
+ """Retry a cached PR flag refresh that failed after its link write committed."""
+ refresh_implementation_pr_cache_for_links(
+ team_id=team_id,
+ report_ids=report_ids or [],
+ pull_request_ids=pull_request_ids or [],
+ task_ids=task_ids or [],
+ )
+
+
@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=5)
@with_team_scope()
def write_pull_request_to_support_tickets(team_id: int, report_id: str, pr_url: str, task_id: str) -> None:
diff --git a/products/signals/backend/test/test_backfill_report_inbox_state.py b/products/signals/backend/test/test_backfill_report_inbox_state.py
new file mode 100644
index 000000000000..c207e3a88857
--- /dev/null
+++ b/products/signals/backend/test/test_backfill_report_inbox_state.py
@@ -0,0 +1,47 @@
+from posthog.test.base import BaseTest
+
+from django.core.management import call_command
+
+from products.signals.backend.models import SignalReport, SignalReportAssignment
+
+
+class TestBackfillReportInboxState(BaseTest):
+ def _report(self) -> SignalReport:
+ return SignalReport.objects.create(team=self.team, status=SignalReport.Status.READY, title="t", summary="s")
+
+ def _flags(self, report: SignalReport) -> tuple[bool, bool]:
+ row = SignalReport.objects.filter(id=report.id).values_list(
+ "has_implementation_pr", "has_active_implementation_pr"
+ )[0]
+ return row[0], row[1]
+
+ def test_corrects_implementation_pr_flags_that_disagree_with_the_links(self) -> None:
+ open_pr = self._report()
+ merged_pr = self._report()
+ stale = self._report()
+ for report, number, merged in ((open_pr, 1, False), (merged_pr, 2, True)):
+ SignalReportAssignment.all_teams.create(
+ team=self.team,
+ report=report,
+ pr_url=f"https://github.com/example/app/pull/{number}",
+ repository="example/app",
+ pr_number=number,
+ pr_state="merged" if merged else "open",
+ pr_merged=merged,
+ )
+ SignalReport.objects.filter(id__in=[open_pr.id, merged_pr.id]).update(
+ has_implementation_pr=False, has_active_implementation_pr=False
+ )
+ SignalReport.objects.filter(id=stale.id).update(has_implementation_pr=True, has_active_implementation_pr=True)
+
+ call_command(
+ "backfill_report_inbox_state",
+ "--team-id",
+ str(self.team.id),
+ "--batch-size",
+ "2",
+ )
+
+ assert self._flags(open_pr) == (True, True)
+ assert self._flags(merged_pr) == (True, False)
+ assert self._flags(stale) == (False, False)
diff --git a/products/signals/backend/test/test_implementation_pr_cache.py b/products/signals/backend/test/test_implementation_pr_cache.py
new file mode 100644
index 000000000000..4549a0ff2347
--- /dev/null
+++ b/products/signals/backend/test/test_implementation_pr_cache.py
@@ -0,0 +1,161 @@
+import json
+from collections.abc import Callable
+
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.db import OperationalError
+
+from parameterized import parameterized
+
+from products.signals.backend.implementation_pr_cache import live_implementation_pr_flags
+from products.signals.backend.models import (
+ SignalReport,
+ SignalReportArtefact,
+ SignalReportAssignment,
+ SignalReportPullRequest,
+ SignalReportTask,
+)
+from products.tasks.backend.models import Task, TaskRun
+
+PR_URL = "https://github.com/example/app/pull/7"
+
+
+class TestImplementationPrCache(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.report = SignalReport.objects.create(
+ team=self.team, status=SignalReport.Status.READY, title="t", summary="s"
+ )
+
+ def _cached_and_live(self) -> tuple[tuple[bool, bool], tuple[bool, bool]]:
+ live = live_implementation_pr_flags(self.team.id)
+ row = (
+ SignalReport.objects.filter(id=self.report.id)
+ .annotate(live_has_implementation_pr=live.has_pr, live_has_active_implementation_pr=live.has_active_pr)
+ .values(
+ "has_implementation_pr",
+ "has_active_implementation_pr",
+ "live_has_implementation_pr",
+ "live_has_active_implementation_pr",
+ )[0]
+ )
+ return (
+ (row["has_implementation_pr"], row["has_active_implementation_pr"]),
+ (row["live_has_implementation_pr"], row["live_has_active_implementation_pr"]),
+ )
+
+ def _task(self, *, with_pr: bool) -> tuple[Task, TaskRun]:
+ task = Task.objects.create(team=self.team, title="Implementation", description="", origin_product="signals")
+ run = TaskRun.objects.create(team=self.team, task=task, status=TaskRun.Status.COMPLETED)
+ if with_pr:
+ # A queryset update, so the run's own receiver is not what links it here.
+ TaskRun.objects.filter(id=run.id).update(output={"pr_url": PR_URL})
+ return task, run
+
+ def _task_run_artefact(self, task: Task, run: TaskRun) -> None:
+ SignalReportArtefact.objects.create(
+ team=self.team,
+ report=self.report,
+ type=SignalReportArtefact.ArtefactType.TASK_RUN,
+ task_id=task.id,
+ content=json.dumps(
+ {"task_id": str(task.id), "run_id": str(run.id), "product": "signals", "type": "implementation"}
+ ),
+ )
+
+ def _link_assignment(self) -> None:
+ SignalReportAssignment.all_teams.create(
+ team=self.team, report=self.report, pr_url=PR_URL, repository="example/app", pr_number=7, pr_state="open"
+ )
+
+ def _link_pull_request(self) -> None:
+ pull_request = SignalReportPullRequest.objects.create(
+ team=self.team, repository="example/app", number=7, url=PR_URL, state="open"
+ )
+ SignalReportArtefact.objects.create(
+ team=self.team,
+ report=self.report,
+ type=SignalReportArtefact.ArtefactType.PULL_REQUEST,
+ pull_request=pull_request,
+ content="{}",
+ )
+
+ def _link_task_run_artefact(self) -> None:
+ self._task_run_artefact(*self._task(with_pr=True))
+
+ def _link_task_run_output(self) -> None:
+ task, run = self._task(with_pr=False)
+ self._task_run_artefact(task, run)
+ run.output = {"pr_url": PR_URL}
+ # The assignment sync on the same save queues tracker linking, which retries without a provider.
+ with patch("products.signals.backend.tasks.link_report_tracker_issues.delay"):
+ run.save(update_fields=["output"])
+
+ def _link_legacy_report_task(self) -> None:
+ task, _ = self._task(with_pr=True)
+ SignalReportTask.objects.create(team=self.team, report=self.report, task=task, relationship="implementation")
+
+ @parameterized.expand(
+ [
+ ("assignment", "_link_assignment"),
+ ("pull_request_artefact", "_link_pull_request"),
+ ("task_run_artefact", "_link_task_run_artefact"),
+ ("task_run_output_written_after_the_link", "_link_task_run_output"),
+ ("legacy_report_task", "_link_legacy_report_task"),
+ ]
+ )
+ def test_each_link_source_keeps_the_cached_flags_equal_to_the_live_filter(self, _name: str, link: str) -> None:
+ write: Callable[[], None] = getattr(self, link)
+ with self.captureOnCommitCallbacks(execute=True):
+ write()
+
+ cached, live = self._cached_and_live()
+ assert live == (True, True)
+ assert cached == live
+
+ def test_a_merged_pull_request_stops_counting_as_active(self) -> None:
+ with self.captureOnCommitCallbacks(execute=True):
+ self._link_pull_request()
+ pull_request = SignalReportPullRequest.objects.get(team=self.team, url=PR_URL)
+
+ with self.captureOnCommitCallbacks(execute=True):
+ pull_request.state = "merged"
+ pull_request.save()
+
+ cached, live = self._cached_and_live()
+ assert live == (True, False)
+ assert cached == live
+
+ @parameterized.expand([("output_cleared", "clear_output"), ("task_deleted", "delete_task")])
+ def test_removing_the_only_task_pr_clears_the_cached_flags(self, _name: str, removal: str) -> None:
+ task, run = self._task(with_pr=True)
+ SignalReportTask.objects.create(team=self.team, report=self.report, task=task, relationship="implementation")
+ self._task_run_artefact(task, run)
+ SignalReport.objects.filter(id=self.report.id).update(
+ has_implementation_pr=True, has_active_implementation_pr=True
+ )
+
+ with self.captureOnCommitCallbacks(execute=True):
+ if removal == "clear_output":
+ run.output = None
+ run.save(update_fields=["output"])
+ else:
+ Task.objects.filter(id=task.id).delete()
+
+ cached, live = self._cached_and_live()
+ assert live == (False, False)
+ assert cached == live
+
+ def test_a_failed_refresh_is_retried_by_a_task(self) -> None:
+ with (
+ patch(
+ "products.signals.backend.implementation_pr_cache.refresh_implementation_pr_cache",
+ side_effect=OperationalError("connection reset"),
+ ),
+ patch("products.signals.backend.tasks.refresh_report_implementation_pr_cache.delay") as retry,
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ self._link_assignment()
+
+ retry.assert_called_once_with(self.team.id, report_ids=[str(self.report.id)], pull_request_ids=[], task_ids=[])