:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): reconcile cached implementation PR flags daily (#124389)
Dylan Martin6e9a06e14665
diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.pyindex ce357e1508bf..e0d3cb02d316 100644--- a/posthog/tasks/scheduled.py+++ b/posthog/tasks/scheduled.py@@ -120,6 +120,7 @@ from products.reminders.backend.tasks import process_due_remindersfrom products.signals.backend.tasks import (pause_inactive_signal_scouts,prune_expired_scratchpad_entries_task,+ reconcile_report_inbox_state_cache,refresh_signal_repository_activity,refresh_signal_scout_background_bands,sweep_first_report_notifications,@@ -463,6 +464,14 @@ def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:name="refresh signals scout background bands",)+ # Correct signals cached implementation PR flags that drifted - daily at 4:40 AM+ add_periodic_task_with_expiry(+ sender,+ crontab(hour="4", minute="40"),+ reconcile_report_inbox_state_cache.s(),+ name="reconcile signals report inbox state",+ )+# Warn, then pause signals scouts that produce nothing anyone uses - daily at 6:15 AMsender.add_periodic_task(crontab(hour="6", minute="15"),diff --git a/products/signals/backend/report_inbox_state.py b/products/signals/backend/report_inbox_state.pynew file mode 100644index 000000000000..2b8000062e55--- /dev/null+++ b/products/signals/backend/report_inbox_state.py@@ -0,0 +1,64 @@+"""Daily check that the cached implementation PR flags still agree with the link tables.++Receivers keep `has_implementation_pr` and `has_active_implementation_pr` current, but a bulk write+that skips Django signals can leave a row stale. This sweep counts those rows as drift, then+corrects them. Sustained zero drift is the evidence that the inbox can read the columns instead of+the live subqueries.+"""++from django.db.models import F, Q++import structlog+from prometheus_client import Counter++from products.signals.backend.implementation_pr_cache import (+ live_implementation_pr_flags,+ refresh_implementation_pr_cache,+)+from products.signals.backend.models import SignalReport+from products.signals.backend.report_actionability_repair import report_id_batches++logger = structlog.get_logger(__name__)++IMPLEMENTATION_PR_CACHE_DRIFT = Counter(+ "signals_implementation_pr_cache_drift_total",+ "Reports whose cached implementation PR flags disagreed with the link tables and were corrected.",+)++_RECONCILE_BATCH_SIZE = 500+++def reconcile_report_inbox_state(team_id: int) -> int:+ """Correct the team's reports whose cached flags disagree with the live filter. Returns how many changed.++ Works through the team one page of report ids at a time, so each comparison narrows the live+ link subqueries to that page and memory stays bounded however many rows drifted.+ """+ corrected = 0+ try:+ for batch in report_id_batches(team_id=team_id, batch_size=_RECONCILE_BATCH_SIZE):+ page = [report_id for report_id, _ in batch.rows]+ live = live_implementation_pr_flags(team_id, report_ids=page)+ stale_ids = list(+ SignalReport.objects.filter(team_id=team_id, id__in=page)+ .annotate(live_has_implementation_pr=live.has_pr, live_has_active_implementation_pr=live.has_active_pr)+ .filter(+ ~Q(has_implementation_pr=F("live_has_implementation_pr"))+ | ~Q(has_active_implementation_pr=F("live_has_active_implementation_pr"))+ )+ .values_list("id", flat=True)+ )+ if not stale_ids:+ continue+ repaired = refresh_implementation_pr_cache(team_id=team_id, report_ids=stale_ids)+ # Counted per committed batch, so a later failure cannot hide repairs that already landed.+ IMPLEMENTATION_PR_CACHE_DRIFT.inc(repaired)+ corrected += repaired+ finally:+ if corrected:+ logger.warning("signals.report_inbox_state.drift_corrected", team_id=team_id, implementation_pr=corrected)+ return corrected+++def teams_with_reports() -> list[int]:+ return list(SignalReport.objects.values_list("team_id", flat=True).distinct().order_by("team_id"))diff --git a/products/signals/backend/tasks.py b/products/signals/backend/tasks.pyindex a0455d571a33..ce8c16f19c51 100644--- a/products/signals/backend/tasks.py+++ b/products/signals/backend/tasks.py@@ -62,6 +62,7 @@ from products.signals.backend.report_generation.repo_activity import (rebuild_repository_activity,repository_activity_needs_rebuild,)+from products.signals.backend.report_inbox_state import reconcile_report_inbox_state, teams_with_reportsfrom products.signals.backend.reviewer_pr_assignment import deliver_pull_request_assignmentfrom products.signals.backend.reviewer_pr_ready import open_pull_request_ready_for_reviewfrom products.signals.backend.scout_harness.background_bands import refresh_background_bands@@ -1263,6 +1264,24 @@ def refresh_signal_scout_background_bands() -> None:logger.info("signals_scout background bands refreshed", written=outcome.written, removed=outcome.removed)+@shared_task(+ name="products.signals.backend.tasks.reconcile_report_inbox_state_cache",+ ignore_result=True,+ max_retries=0,+)+@skip_team_scope_audit+def reconcile_report_inbox_state_cache() -> None:+ """Daily: correct cached implementation PR flags that drifted from the link tables, and count the drift."""+ corrected = failed = 0+ for team_id in teams_with_reports():+ try:+ corrected += reconcile_report_inbox_state(team_id)+ except Exception as error:+ failed += 1+ capture_exception(error, {"team_id": team_id})+ logger.info("signals report inbox state reconciled", implementation_pr=corrected, failed_teams=failed)++@shared_task(name="products.signals.backend.tasks.pause_inactive_signal_scouts",ignore_result=True,diff --git a/products/signals/backend/test/test_backfill_report_inbox_state.py b/products/signals/backend/test/test_backfill_report_inbox_state.pyindex c207e3a88857..6b33bf5ebad6 100644--- a/products/signals/backend/test/test_backfill_report_inbox_state.py+++ b/products/signals/backend/test/test_backfill_report_inbox_state.py@@ -1,8 +1,10 @@from posthog.test.base import BaseTest+from unittest.mock import patchfrom django.core.management import call_commandfrom products.signals.backend.models import SignalReport, SignalReportAssignment+from products.signals.backend.report_inbox_state import reconcile_report_inbox_stateclass TestBackfillReportInboxState(BaseTest):@@ -45,3 +47,24 @@ class TestBackfillReportInboxState(BaseTest):assert self._flags(open_pr) == (True, True)assert self._flags(merged_pr) == (True, False)assert self._flags(stale) == (False, False)++ def test_reconcile_counts_and_corrects_rows_a_bulk_write_left_stale(self) -> None:+ linked = [self._report(), self._report()]+ SignalReportAssignment.all_teams.bulk_create(+ [+ SignalReportAssignment(+ team=self.team,+ report=report,+ pr_url=f"https://github.com/example/app/pull/{number}",+ repository="example/app",+ pr_number=number,+ pr_state="open",+ )+ for number, report in enumerate(linked, start=3)+ ]+ )++ with patch("products.signals.backend.report_inbox_state._RECONCILE_BATCH_SIZE", 1):+ assert reconcile_report_inbox_state(self.team.id) == 2+ assert [self._flags(report) for report in linked] == [(True, True), (True, True)]+ assert reconcile_report_inbox_state(self.team.id) == 0