:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): author monitoring check cadence (#119955)
Mikayla Thompson7c5b8caea3e5
diff --git a/docs/internal/signals-pr-lifecycle.md b/docs/internal/signals-pr-lifecycle.mdindex d60b9ee4d7cb..6f8daa7c5cde 100644--- a/docs/internal/signals-pr-lifecycle.md+++ b/docs/internal/signals-pr-lifecycle.md@@ -67,6 +67,8 @@ With `signals-report-metrics` enabled, research gathers short-window baselines bThe server refuses new Monitoring definitions when `signals-report-monitoring` is disabled or unavailable and omits the definition from check responses. Stored definitions remain intact. Research carries its definition-visibility decision into reconciliation, so flag recovery during a run does not replace an unchanged check or reset its approval and schedule. Existing definitions, charts, check schedules, and Resolved outcomes otherwise follow their current behavior.+Enabled research can author `run_interval_minutes` for a metric check with a Monitoring definition. The interval must be at least six hours and strictly shorter than the check's observation window. A new check without an interval defaults to a quarter of that window, bounded to six hours through one day. A 24-hour window therefore stores a six-hour cadence, allowing interim measurements at hours 6, 12, and 18 when interim execution is available. Windows of six hours or less retain their evidence-backed duration and have no interim samples. Cadence is independent of query bucket size and the target reference period. Omitted intervals preserve existing checks' schedules; explicit cadence revisions replace the check and clear approval. Revised windows must still accommodate inherited intervals. This authoring policy does not change existing null-interval checks, execution, or final outcomes.+## Inbox value summaryThe `signals-inbox-value-summary` flag shows project-wide counts above the webdiff --git a/products/signals/backend/report_check_authoring.py b/products/signals/backend/report_check_authoring.pyindex 164e66ac2c62..b3c26719d5ce 100644--- a/products/signals/backend/report_check_authoring.py+++ b/products/signals/backend/report_check_authoring.py@@ -43,6 +43,7 @@ from products.signals.backend.report_checks import (CheckConfigValidationError,CheckSpec,MetricThresholdConfig,+ authored_monitoring_interval_minutes,check_schedule_expires_at,parse_check_config,soak_minutes_from_gap,@@ -315,6 +316,26 @@ class CheckReconciliation:created: list[SignalReportCheck]+def _spec_soak_minutes(spec: CheckSpec, previous: SignalReportCheck | None) -> int:+ if previous is not None and (+ "soak_hours" not in spec.model_fields_set+ or spec.soak_hours == max(1, round((previous.soak_minutes or 60) / 60))+ ):+ return previous.soak_minutes if previous.soak_minutes is not None else DEFAULT_CHECK_SOAK_HOURS * 60+ return spec.soak_hours * 60+++def _spec_interval_minutes(spec: CheckSpec, previous: SignalReportCheck | None) -> int | None:+ interval = spec.run_interval_minutes+ if interval is None and previous is not None:+ interval = previous.run_interval_minutes+ if interval is None:+ return None+ if spec.config.get("monitoring") is not None:+ return authored_monitoring_interval_minutes(_spec_soak_minutes(spec, previous), interval)+ return interval++def create_checks_from_specs(*,report: SignalReport,@@ -377,6 +398,9 @@ def create_checks_from_specs(and check.title == spec.titleand check.rationale == spec.rationaleand check.kind == spec.kind+ and (+ spec.run_interval_minutes is None or check.run_interval_minutes == spec.run_interval_minutes+ )and ("soak_hours" not in spec.model_fields_setor max(1, round((check.soak_minutes or 60) / 60)) == spec.soak_hours@@ -402,16 +426,8 @@ def create_checks_from_specs(kind=spec.kind,config=spec.config,attribution=attribution,- soak_minutes=(- previous.soak_minutes if previous.soak_minutes is not None else DEFAULT_CHECK_SOAK_HOURS * 60- )- if previous is not None- and (- "soak_hours" not in spec.model_fields_set- or spec.soak_hours == max(1, round((previous.soak_minutes or 60) / 60))- )- else spec.soak_hours * 60,- run_interval_minutes=previous.run_interval_minutes if previous is not None else None,+ soak_minutes=_spec_soak_minutes(spec, previous),+ run_interval_minutes=_spec_interval_minutes(spec, previous),runs_remaining=previous.runs_remaining if previous is not None else 1,)for spec, previous in new_specsdiff --git a/products/signals/backend/report_checks.py b/products/signals/backend/report_checks.pyindex 3731ffdd82ae..5f3eba0a9da6 100644--- a/products/signals/backend/report_checks.py+++ b/products/signals/backend/report_checks.py@@ -354,6 +354,18 @@ class CheckConfigValidationError(ValueError):"""A check config that does not match its kind's schema."""+def authored_monitoring_interval_minutes(soak_minutes: int, run_interval_minutes: int | None) -> int | None:+ if run_interval_minutes is not None:+ if not MIN_CHECK_INTERVAL_MINUTES <= run_interval_minutes < soak_minutes:+ raise CheckConfigValidationError(+ "A Monitoring interval must be at least six hours and shorter than the observation window."+ )+ return run_interval_minutes+ if soak_minutes <= MIN_CHECK_INTERVAL_MINUTES:+ return None+ return min(24 * 60, max(MIN_CHECK_INTERVAL_MINUTES, soak_minutes // 4))++def parse_check_config(kind: str, config: object) -> BaseModel:"""Parse a raw config payload into its kind's model.@@ -453,6 +465,17 @@ class CheckSpec(BaseModel):"for a new check; omission preserves an existing check’s wait."),)+ run_interval_minutes: int | None = Field(+ default=None,+ strict=True,+ ge=MIN_CHECK_INTERVAL_MINUTES,+ le=MAX_CHECK_INTERVAL_MINUTES,+ description=(+ "Minutes between interim Monitoring measurements. Must be shorter than soak_hours * 60. "+ "Omission preserves an existing check's interval. New Monitoring metric checks default to a quarter "+ "of the window, bounded to six hours through one day. Windows of six hours or less have no interim samples."+ ),+ )@field_validator("title")@classmethod@@ -464,5 +487,8 @@ class CheckSpec(BaseModel):@model_validator(mode="after")def config_must_match_its_kind(self) -> CheckSpec:- parse_check_config(self.kind, self.config)+ config = parse_check_config(self.kind, self.config)+ if self.run_interval_minutes is not None:+ if not isinstance(config, MetricThresholdConfig) or config.monitoring is None:+ raise ValueError("Research can author an interval only for a Monitoring metric check.")return selfdiff --git a/products/signals/backend/report_generation/research.py b/products/signals/backend/report_generation/research.pyindex 90d5a8eee635..3112d51457ba 100644--- a/products/signals/backend/report_generation/research.py+++ b/products/signals/backend/report_generation/research.py@@ -415,7 +415,15 @@ _MONITORING_METRIC_CHECK_GUIDANCE = """- Make every effort to complete `config.mIf required facts remain unknown, explain what is missing in the rationale and omit monitoring ratherthan invent them. Preserve the bounded relative queryand whole-window comparison for existing Resolved behavior. This only defines the measurement; it does- not enable interim execution or decide a final outcome."""+ not enable interim execution or decide a final outcome.+- For metric checks with `config.monitoring`, author `run_interval_minutes` based on activity and+ ingestion delay. It controls interim measurement cadence, independently of query bucket size and+ target reference period. Use at least 360 minutes and strictly less than `soak_hours * 60`.+ A 24-hour window with a 360-minute interval allows measurements at hours 6, 12, and 18.+ Omission preserves an existing check's cadence. New checks default to a quarter of the window,+ bounded to six hours through one day. For windows of six hours or less, omit the interval:+ no interim sample fits. Preserve the evidence-backed window instead of lengthening it to fit+ a sample. Interim measurements do not decide the final outcome. Explain the timing in the rationale."""_AGENT_CHECK_GUIDANCE = """- {agent_selection} It needs no metric,baseline, or comparison. The `config` is@@ -1425,6 +1433,8 @@ def build_fix_verification_prompt(else "Do not force a product metric when another method gives better evidence.")schema_dict = FixVerificationOutput.model_json_schema()+ if not (metric_checks_enabled and monitoring_enabled):+ schema_dict["$defs"]["CheckSpec"]["properties"].pop("run_interval_minutes", None)kinds = [guidancefor guidance, enabled in (diff --git a/products/signals/backend/test/test_report_checks.py b/products/signals/backend/test/test_report_checks.pyindex 0b26a3735f1d..e1bf44a999f9 100644--- a/products/signals/backend/test/test_report_checks.py+++ b/products/signals/backend/test/test_report_checks.py@@ -87,6 +87,7 @@ from products.signals.backend.report_checks import (CheckConfigValidationError,CheckSpec,MetricThresholdConfig,+ authored_monitoring_interval_minutes,parse_check_config,validate_metric_check_for_write,)@@ -462,6 +463,31 @@ class TestAgentCheckConfig(SimpleTestCase):class TestCheckScheduleValidation(SimpleTestCase):+ @parameterized.expand(+ [("metric_threshold", _threshold_config()), ("agent", {"instructions": "Check the issue again."})]+ )+ def test_research_cannot_add_monitoring_cadence_to_other_checks(self, kind: str, config: dict[str, object]) -> None:+ with self.assertRaises(PydanticValidationError):+ CheckSpec.model_validate(+ {"title": "Check recovery", "kind": kind, "config": config, "run_interval_minutes": 360}+ )++ @parameterized.expand(+ [+ ("short", 4, None, None),+ ("exact_floor", 6, None, None),+ ("one_sample", 8, None, 360),+ ("daily", 24, None, 360),+ ("two_days", 48, None, 720),+ ("weekly", 168, None, 1440),+ ("authored", 24, 480, 480),+ ]+ )+ def test_monitoring_cadence_fits_the_window(+ self, _name: str, soak_hours: int, interval: int | None, expected: int | None+ ) -> None:+ assert authored_monitoring_interval_minutes(soak_hours * 60, interval) == expected+@parameterized.expand([("first_run_in_the_past", {"next_run_at": "2020-01-01T00:00:00Z"}, "next_run_at"),@@ -2898,20 +2924,90 @@ class TestResearchAuthoredChecks(APIBaseTest):payload.update(overrides)return CheckSpec.model_validate(payload)- def test_a_spec_becomes_a_pending_check_carrying_the_metrics_query(self) -> None:- with patch(_CAPTURE) as capture, self.captureOnCommitCallbacks(execute=True):- written = create_checks_from_specs(- report=self.report, specs=[self._spec()], attribution=ArtefactAttribution.system()- ).created+ @parameterized.expand(+ [("legacy", False, False), ("enabled", True, True), ("disabled", True, False), ("unavailable", True, None)]+ )+ def test_a_spec_becomes_a_pending_check_carrying_the_metrics_query(+ self, _name: str, monitoring: bool, flag_result: bool | None+ ) -> None:+ spec = self._spec()+ if monitoring:+ spec.config["monitoring"] = {+ "reference_period_minutes": 1440,+ "threshold_scaling": "fixed",+ "baseline_period_minutes": 1440,+ }+ with (+ patch(_CAPTURE) as capture,+ patch("products.signals.backend.report_content_gates.feature_enabled_or_false", return_value=flag_result),+ self.captureOnCommitCallbacks(execute=True),+ ):+ result = create_checks_from_specs(+ report=self.report, specs=[spec], attribution=ArtefactAttribution.system()+ )+ if monitoring and flag_result is not True:+ assert not result.applied+ assert not self.report.checks.exists()+ return+ written = result.createdassert len(written) == 1check = written[0]assert check.status == SignalReportCheck.Status.PENDINGassert check.soak_minutes == DEFAULT_CHECK_SOAK_HOURS * 60+ assert check.run_interval_minutes == (360 if monitoring else None)assert check.config["query"] == _PAGEVIEWSassert capture.call_args.kwargs["event"] == "signals_report_check_created"assert capture.call_args.kwargs["properties"]["check_status"] == SignalReportCheck.Status.PENDING+ @parameterized.expand(+ [+ ("omitted", {}, True, False),+ ("unchanged", {"run_interval_minutes": 1440}, True, False),+ ("revised", {"run_interval_minutes": 720}, True, True),+ ("inherited_interval_does_not_fit", {"soak_hours": 12}, False, False),+ ("interval_equals_window", {"soak_hours": 24}, False, False),+ ("revised_window_and_interval", {"soak_hours": 12, "run_interval_minutes": 360}, True, True),+ ]+ )+ def test_monitoring_cadence_revisions_preserve_checks_until_valid(+ self, _name: str, overrides: dict[str, int], applied: bool, replaced: bool+ ) -> None:+ config = self._spec().config+ config["monitoring"] = {+ "reference_period_minutes": 1440,+ "threshold_scaling": "fixed",+ "baseline_period_minutes": 1440,+ }+ spec = self._spec(config=config, soak_hours=48, run_interval_minutes=1440)+ with patch("products.signals.backend.report_content_gates.feature_enabled_or_false", return_value=True):+ existing = create_checks_from_specs(+ report=self.report, specs=[spec], attribution=ArtefactAttribution.system()+ ).created[0]+ existing.approved_at = timezone.now()+ existing.save(update_fields=["approved_at", "updated_at"])+ version = existing.updated_at+ revised = self._spec(config=spec.config, existing_check_id=existing.id, **overrides)+ result = create_checks_from_specs(+ report=self.report,+ specs=[revised],+ attribution=ArtefactAttribution.system(),+ checks_snapshot=check_versions([existing]),+ )++ existing.refresh_from_db()+ assert result.applied is applied+ assert existing.status == (SignalReportCheck.Status.CANCELLED if replaced else SignalReportCheck.Status.PENDING)+ if replaced:+ assert result.created[0].run_interval_minutes == overrides["run_interval_minutes"]+ assert result.created[0].soak_minutes == overrides.get("soak_hours", 48) * 60+ assert result.created[0].approved_at is None+ assert result.created[0].runs_remaining == existing.runs_remaining+ else:+ assert not result.created+ assert existing.updated_at == version+ assert existing.approved_at is not None+@parameterized.expand([("no_lane", False), ("lane", True)])def test_research_drops_an_agent_spec_with_no_lane_and_keeps_the_rest(self, _name: str, has_lane: bool) -> None:if has_lane:@@ -2955,6 +3051,7 @@ class TestResearchAuthoredChecks(APIBaseTest):("hidden_monitoring_definition", False, "unchanged", "monitoring_disabled"),("hidden_monitoring_after_flag_recovery", False, "unchanged", "monitoring_recovered"),("visible_monitoring_removed", False, "revise", "monitoring_visible"),+ ("revise_monitoring_with_null_interval", False, "revise", "monitoring_null_interval"),("config_written_before_display_fields", True, "unchanged"),("revise_approved", False, "revise"),("retire_approved", False, "retire"),@@ -2964,6 +3061,7 @@ class TestResearchAuthoredChecks(APIBaseTest):("omitted_existing_wait", False, "unchanged", "omitted_wait"),("revise_omitted_existing_wait", False, "revise", "omitted_wait"),("longer_wait", False, "revise", "longer_wait"),+ ("revise_legacy_null_wait", False, "revise", "legacy_null_wait"),("invalid_stored_config", False, "revise", "invalid_stored"),])@@ -2974,7 +3072,7 @@ class TestResearchAuthoredChecks(APIBaseTest):spec = self._spec()if variant == "metric_defaults":spec.config["baseline_value"] = None- elif variant == "agent_defaults":+ elif variant in {"agent_defaults", "legacy_null_wait"}:spec = self._spec(kind="agent", config={"instructions": "Check the issue again."})existing = create_checks_from_specs(report=self.report, specs=[spec], attribution=ArtefactAttribution.system()@@ -2988,7 +3086,7 @@ class TestResearchAuthoredChecks(APIBaseTest):stored_config.update(probe_hints=[], skill_name=None)if variant == "invalid_stored":stored_config["retired_config_field"] = "obsolete"- if variant in {"monitoring_disabled", "monitoring_recovered", "monitoring_visible"}:+ if variant in {"monitoring_disabled", "monitoring_recovered", "monitoring_visible", "monitoring_null_interval"}:stored_config["monitoring"] = {"reference_period_minutes": 10080,"threshold_scaling": "fixed",@@ -2996,6 +3094,8 @@ class TestResearchAuthoredChecks(APIBaseTest):}if variant == "omitted_wait":SignalReportCheck.objects.for_team(self.team.id).filter(id=existing.id).update(soak_minutes=72 * 60)+ if variant == "legacy_null_wait":+ SignalReportCheck.objects.for_team(self.team.id).filter(id=existing.id).update(soak_minutes=None)if variant == "recurring":SignalReportCheck.objects.for_team(self.team.id).filter(id=existing.id).update(soak_minutes=1450, run_interval_minutes=7 * 24 * 60, runs_remaining=3@@ -3019,6 +3119,14 @@ class TestResearchAuthoredChecks(APIBaseTest):if variant == "monitoring_visible":specs = [spec]+ if variant == "legacy_null_wait":+ specs = [+ self._spec(+ title="Revised goal", kind="agent", config=spec.config, existing_check_id=existing.id, soak_hours=1+ )+ ]+ if variant == "monitoring_null_interval":+ specs[0].config["monitoring"] = stored_config["monitoring"]if variant == "omitted_wait":specs[0] = self._spec(title=specs[0].title, existing_check_id=existing.id)if variant == "invalid_stored":@@ -3027,7 +3135,7 @@ class TestResearchAuthoredChecks(APIBaseTest):specs[0] = specs[0].model_copy(update={"soak_hours": 72})with patch("products.signals.backend.report_content_gates.feature_enabled_or_false",- return_value=variant in {"monitoring_recovered", "monitoring_visible"},+ return_value=variant in {"monitoring_recovered", "monitoring_visible", "monitoring_null_interval"},):written = create_checks_from_specs(report=self.report,@@ -3053,7 +3161,13 @@ class TestResearchAuthoredChecks(APIBaseTest):if written:assert written[0].title == specs[0].titleassert written[0].approved_at is None- assert written[0].soak_minutes == (72 * 60 if variant == "longer_wait" else existing.soak_minutes)+ assert written[0].soak_minutes == (+ DEFAULT_CHECK_SOAK_HOURS * 60+ if variant == "legacy_null_wait"+ else 72 * 60+ if variant == "longer_wait"+ else existing.soak_minutes+ )assert written[0].run_interval_minutes == existing.run_interval_minutesassert written[0].runs_remaining == existing.runs_remainingassert SignalReportCheck.objects.for_team(self.team.id).filter(report=self.report).count() == (diff --git a/products/signals/backend/test/test_research_prompt.py b/products/signals/backend/test/test_research_prompt.pyindex 6fe0110b6932..4a11805665ff 100644--- a/products/signals/backend/test/test_research_prompt.py+++ b/products/signals/backend/test/test_research_prompt.py@@ -299,7 +299,12 @@ class TestBuildFixVerificationPrompt:agent_checks_enabled=agent_checks_enabled,)preferred = metric_checks_enabled- for field in ("reference_period_minutes", "threshold_scaling", "baseline_period_minutes"):+ for field in (+ "reference_period_minutes",+ "threshold_scaling",+ "baseline_period_minutes",+ "run_interval_minutes",+ ):assert (field in prompt) is (metric_checks_enabled and monitoring_enabled)assert ("70-80% confidence" in prompt) is preferredassert ('Prefer `kind: "metric_threshold"`' in prompt) is preferred