Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): author monitoring check cadence (#119955)

Mikayla Thompson7c5b8caea3e5
diff --git a/docs/internal/signals-pr-lifecycle.md b/docs/internal/signals-pr-lifecycle.md
index d60b9ee4d7cb..6f8daa7c5cde 100644
--- a/docs/internal/signals-pr-lifecycle.md
+++ b/docs/internal/signals-pr-lifecycle.md
@@ -67,6 +67,8 @@ With `signals-report-metrics` enabled, research gathers short-window baselines b
The server refuses new Monitoring definitions when `signals-report-monitoring` is disabled or unavailable and omits the definition from check responses. Stored definitions remain intact. Research carries its definition-visibility decision into reconciliation, so flag recovery during a run does not replace an unchanged check or reset its approval and schedule. Existing definitions, charts, check schedules, and Resolved outcomes otherwise follow their current behavior.
+Enabled research can author `run_interval_minutes` for a metric check with a Monitoring definition. The interval must be at least six hours and strictly shorter than the check's observation window. A new check without an interval defaults to a quarter of that window, bounded to six hours through one day. A 24-hour window therefore stores a six-hour cadence, allowing interim measurements at hours 6, 12, and 18 when interim execution is available. Windows of six hours or less retain their evidence-backed duration and have no interim samples. Cadence is independent of query bucket size and the target reference period. Omitted intervals preserve existing checks' schedules; explicit cadence revisions replace the check and clear approval. Revised windows must still accommodate inherited intervals. This authoring policy does not change existing null-interval checks, execution, or final outcomes.
+
## Inbox value summary
The `signals-inbox-value-summary` flag shows project-wide counts above the web
diff --git a/products/signals/backend/report_check_authoring.py b/products/signals/backend/report_check_authoring.py
index 164e66ac2c62..b3c26719d5ce 100644
--- a/products/signals/backend/report_check_authoring.py
+++ b/products/signals/backend/report_check_authoring.py
@@ -43,6 +43,7 @@ from products.signals.backend.report_checks import (
CheckConfigValidationError,
CheckSpec,
MetricThresholdConfig,
+ authored_monitoring_interval_minutes,
check_schedule_expires_at,
parse_check_config,
soak_minutes_from_gap,
@@ -315,6 +316,26 @@ class CheckReconciliation:
created: list[SignalReportCheck]
+def _spec_soak_minutes(spec: CheckSpec, previous: SignalReportCheck | None) -> int:
+ if previous is not None and (
+ "soak_hours" not in spec.model_fields_set
+ or spec.soak_hours == max(1, round((previous.soak_minutes or 60) / 60))
+ ):
+ return previous.soak_minutes if previous.soak_minutes is not None else DEFAULT_CHECK_SOAK_HOURS * 60
+ return spec.soak_hours * 60
+
+
+def _spec_interval_minutes(spec: CheckSpec, previous: SignalReportCheck | None) -> int | None:
+ interval = spec.run_interval_minutes
+ if interval is None and previous is not None:
+ interval = previous.run_interval_minutes
+ if interval is None:
+ return None
+ if spec.config.get("monitoring") is not None:
+ return authored_monitoring_interval_minutes(_spec_soak_minutes(spec, previous), interval)
+ return interval
+
+
def create_checks_from_specs(
*,
report: SignalReport,
@@ -377,6 +398,9 @@ def create_checks_from_specs(
and check.title == spec.title
and check.rationale == spec.rationale
and check.kind == spec.kind
+ and (
+ spec.run_interval_minutes is None or check.run_interval_minutes == spec.run_interval_minutes
+ )
and (
"soak_hours" not in spec.model_fields_set
or max(1, round((check.soak_minutes or 60) / 60)) == spec.soak_hours
@@ -402,16 +426,8 @@ def create_checks_from_specs(
kind=spec.kind,
config=spec.config,
attribution=attribution,
- soak_minutes=(
- previous.soak_minutes if previous.soak_minutes is not None else DEFAULT_CHECK_SOAK_HOURS * 60
- )
- if previous is not None
- and (
- "soak_hours" not in spec.model_fields_set
- or spec.soak_hours == max(1, round((previous.soak_minutes or 60) / 60))
- )
- else spec.soak_hours * 60,
- run_interval_minutes=previous.run_interval_minutes if previous is not None else None,
+ soak_minutes=_spec_soak_minutes(spec, previous),
+ run_interval_minutes=_spec_interval_minutes(spec, previous),
runs_remaining=previous.runs_remaining if previous is not None else 1,
)
for spec, previous in new_specs
diff --git a/products/signals/backend/report_checks.py b/products/signals/backend/report_checks.py
index 3731ffdd82ae..5f3eba0a9da6 100644
--- a/products/signals/backend/report_checks.py
+++ b/products/signals/backend/report_checks.py
@@ -354,6 +354,18 @@ class CheckConfigValidationError(ValueError):
"""A check config that does not match its kind's schema."""
+def authored_monitoring_interval_minutes(soak_minutes: int, run_interval_minutes: int | None) -> int | None:
+ if run_interval_minutes is not None:
+ if not MIN_CHECK_INTERVAL_MINUTES <= run_interval_minutes < soak_minutes:
+ raise CheckConfigValidationError(
+ "A Monitoring interval must be at least six hours and shorter than the observation window."
+ )
+ return run_interval_minutes
+ if soak_minutes <= MIN_CHECK_INTERVAL_MINUTES:
+ return None
+ return min(24 * 60, max(MIN_CHECK_INTERVAL_MINUTES, soak_minutes // 4))
+
+
def parse_check_config(kind: str, config: object) -> BaseModel:
"""Parse a raw config payload into its kind's model.
@@ -453,6 +465,17 @@ class CheckSpec(BaseModel):
"for a new check; omission preserves an existing check’s wait."
),
)
+ run_interval_minutes: int | None = Field(
+ default=None,
+ strict=True,
+ ge=MIN_CHECK_INTERVAL_MINUTES,
+ le=MAX_CHECK_INTERVAL_MINUTES,
+ description=(
+ "Minutes between interim Monitoring measurements. Must be shorter than soak_hours * 60. "
+ "Omission preserves an existing check's interval. New Monitoring metric checks default to a quarter "
+ "of the window, bounded to six hours through one day. Windows of six hours or less have no interim samples."
+ ),
+ )
@field_validator("title")
@classmethod
@@ -464,5 +487,8 @@ class CheckSpec(BaseModel):
@model_validator(mode="after")
def config_must_match_its_kind(self) -> CheckSpec:
- parse_check_config(self.kind, self.config)
+ config = parse_check_config(self.kind, self.config)
+ if self.run_interval_minutes is not None:
+ if not isinstance(config, MetricThresholdConfig) or config.monitoring is None:
+ raise ValueError("Research can author an interval only for a Monitoring metric check.")
return self
diff --git a/products/signals/backend/report_generation/research.py b/products/signals/backend/report_generation/research.py
index 90d5a8eee635..3112d51457ba 100644
--- a/products/signals/backend/report_generation/research.py
+++ b/products/signals/backend/report_generation/research.py
@@ -415,7 +415,15 @@ _MONITORING_METRIC_CHECK_GUIDANCE = """- Make every effort to complete `config.m
If required facts remain unknown, explain what is missing in the rationale and omit monitoring rather
than invent them. Preserve the bounded relative query
and whole-window comparison for existing Resolved behavior. This only defines the measurement; it does
- not enable interim execution or decide a final outcome."""
+ not enable interim execution or decide a final outcome.
+- For metric checks with `config.monitoring`, author `run_interval_minutes` based on activity and
+ ingestion delay. It controls interim measurement cadence, independently of query bucket size and
+ target reference period. Use at least 360 minutes and strictly less than `soak_hours * 60`.
+ A 24-hour window with a 360-minute interval allows measurements at hours 6, 12, and 18.
+ Omission preserves an existing check's cadence. New checks default to a quarter of the window,
+ bounded to six hours through one day. For windows of six hours or less, omit the interval:
+ no interim sample fits. Preserve the evidence-backed window instead of lengthening it to fit
+ a sample. Interim measurements do not decide the final outcome. Explain the timing in the rationale."""
_AGENT_CHECK_GUIDANCE = """- {agent_selection} It needs no metric,
baseline, or comparison. The `config` is
@@ -1425,6 +1433,8 @@ def build_fix_verification_prompt(
else "Do not force a product metric when another method gives better evidence."
)
schema_dict = FixVerificationOutput.model_json_schema()
+ if not (metric_checks_enabled and monitoring_enabled):
+ schema_dict["$defs"]["CheckSpec"]["properties"].pop("run_interval_minutes", None)
kinds = [
guidance
for guidance, enabled in (
diff --git a/products/signals/backend/test/test_report_checks.py b/products/signals/backend/test/test_report_checks.py
index 0b26a3735f1d..e1bf44a999f9 100644
--- a/products/signals/backend/test/test_report_checks.py
+++ b/products/signals/backend/test/test_report_checks.py
@@ -87,6 +87,7 @@ from products.signals.backend.report_checks import (
CheckConfigValidationError,
CheckSpec,
MetricThresholdConfig,
+ authored_monitoring_interval_minutes,
parse_check_config,
validate_metric_check_for_write,
)
@@ -462,6 +463,31 @@ class TestAgentCheckConfig(SimpleTestCase):
class TestCheckScheduleValidation(SimpleTestCase):
+ @parameterized.expand(
+ [("metric_threshold", _threshold_config()), ("agent", {"instructions": "Check the issue again."})]
+ )
+ def test_research_cannot_add_monitoring_cadence_to_other_checks(self, kind: str, config: dict[str, object]) -> None:
+ with self.assertRaises(PydanticValidationError):
+ CheckSpec.model_validate(
+ {"title": "Check recovery", "kind": kind, "config": config, "run_interval_minutes": 360}
+ )
+
+ @parameterized.expand(
+ [
+ ("short", 4, None, None),
+ ("exact_floor", 6, None, None),
+ ("one_sample", 8, None, 360),
+ ("daily", 24, None, 360),
+ ("two_days", 48, None, 720),
+ ("weekly", 168, None, 1440),
+ ("authored", 24, 480, 480),
+ ]
+ )
+ def test_monitoring_cadence_fits_the_window(
+ self, _name: str, soak_hours: int, interval: int | None, expected: int | None
+ ) -> None:
+ assert authored_monitoring_interval_minutes(soak_hours * 60, interval) == expected
+
@parameterized.expand(
[
("first_run_in_the_past", {"next_run_at": "2020-01-01T00:00:00Z"}, "next_run_at"),
@@ -2898,20 +2924,90 @@ class TestResearchAuthoredChecks(APIBaseTest):
payload.update(overrides)
return CheckSpec.model_validate(payload)
- def test_a_spec_becomes_a_pending_check_carrying_the_metrics_query(self) -> None:
- with patch(_CAPTURE) as capture, self.captureOnCommitCallbacks(execute=True):
- written = create_checks_from_specs(
- report=self.report, specs=[self._spec()], attribution=ArtefactAttribution.system()
- ).created
+ @parameterized.expand(
+ [("legacy", False, False), ("enabled", True, True), ("disabled", True, False), ("unavailable", True, None)]
+ )
+ def test_a_spec_becomes_a_pending_check_carrying_the_metrics_query(
+ self, _name: str, monitoring: bool, flag_result: bool | None
+ ) -> None:
+ spec = self._spec()
+ if monitoring:
+ spec.config["monitoring"] = {
+ "reference_period_minutes": 1440,
+ "threshold_scaling": "fixed",
+ "baseline_period_minutes": 1440,
+ }
+ with (
+ patch(_CAPTURE) as capture,
+ patch("products.signals.backend.report_content_gates.feature_enabled_or_false", return_value=flag_result),
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ result = create_checks_from_specs(
+ report=self.report, specs=[spec], attribution=ArtefactAttribution.system()
+ )
+ if monitoring and flag_result is not True:
+ assert not result.applied
+ assert not self.report.checks.exists()
+ return
+ written = result.created
assert len(written) == 1
check = written[0]
assert check.status == SignalReportCheck.Status.PENDING
assert check.soak_minutes == DEFAULT_CHECK_SOAK_HOURS * 60
+ assert check.run_interval_minutes == (360 if monitoring else None)
assert check.config["query"] == _PAGEVIEWS
assert capture.call_args.kwargs["event"] == "signals_report_check_created"
assert capture.call_args.kwargs["properties"]["check_status"] == SignalReportCheck.Status.PENDING
+ @parameterized.expand(
+ [
+ ("omitted", {}, True, False),
+ ("unchanged", {"run_interval_minutes": 1440}, True, False),
+ ("revised", {"run_interval_minutes": 720}, True, True),
+ ("inherited_interval_does_not_fit", {"soak_hours": 12}, False, False),
+ ("interval_equals_window", {"soak_hours": 24}, False, False),
+ ("revised_window_and_interval", {"soak_hours": 12, "run_interval_minutes": 360}, True, True),
+ ]
+ )
+ def test_monitoring_cadence_revisions_preserve_checks_until_valid(
+ self, _name: str, overrides: dict[str, int], applied: bool, replaced: bool
+ ) -> None:
+ config = self._spec().config
+ config["monitoring"] = {
+ "reference_period_minutes": 1440,
+ "threshold_scaling": "fixed",
+ "baseline_period_minutes": 1440,
+ }
+ spec = self._spec(config=config, soak_hours=48, run_interval_minutes=1440)
+ with patch("products.signals.backend.report_content_gates.feature_enabled_or_false", return_value=True):
+ existing = create_checks_from_specs(
+ report=self.report, specs=[spec], attribution=ArtefactAttribution.system()
+ ).created[0]
+ existing.approved_at = timezone.now()
+ existing.save(update_fields=["approved_at", "updated_at"])
+ version = existing.updated_at
+ revised = self._spec(config=spec.config, existing_check_id=existing.id, **overrides)
+ result = create_checks_from_specs(
+ report=self.report,
+ specs=[revised],
+ attribution=ArtefactAttribution.system(),
+ checks_snapshot=check_versions([existing]),
+ )
+
+ existing.refresh_from_db()
+ assert result.applied is applied
+ assert existing.status == (SignalReportCheck.Status.CANCELLED if replaced else SignalReportCheck.Status.PENDING)
+ if replaced:
+ assert result.created[0].run_interval_minutes == overrides["run_interval_minutes"]
+ assert result.created[0].soak_minutes == overrides.get("soak_hours", 48) * 60
+ assert result.created[0].approved_at is None
+ assert result.created[0].runs_remaining == existing.runs_remaining
+ else:
+ assert not result.created
+ assert existing.updated_at == version
+ assert existing.approved_at is not None
+
@parameterized.expand([("no_lane", False), ("lane", True)])
def test_research_drops_an_agent_spec_with_no_lane_and_keeps_the_rest(self, _name: str, has_lane: bool) -> None:
if has_lane:
@@ -2955,6 +3051,7 @@ class TestResearchAuthoredChecks(APIBaseTest):
("hidden_monitoring_definition", False, "unchanged", "monitoring_disabled"),
("hidden_monitoring_after_flag_recovery", False, "unchanged", "monitoring_recovered"),
("visible_monitoring_removed", False, "revise", "monitoring_visible"),
+ ("revise_monitoring_with_null_interval", False, "revise", "monitoring_null_interval"),
("config_written_before_display_fields", True, "unchanged"),
("revise_approved", False, "revise"),
("retire_approved", False, "retire"),
@@ -2964,6 +3061,7 @@ class TestResearchAuthoredChecks(APIBaseTest):
("omitted_existing_wait", False, "unchanged", "omitted_wait"),
("revise_omitted_existing_wait", False, "revise", "omitted_wait"),
("longer_wait", False, "revise", "longer_wait"),
+ ("revise_legacy_null_wait", False, "revise", "legacy_null_wait"),
("invalid_stored_config", False, "revise", "invalid_stored"),
]
)
@@ -2974,7 +3072,7 @@ class TestResearchAuthoredChecks(APIBaseTest):
spec = self._spec()
if variant == "metric_defaults":
spec.config["baseline_value"] = None
- elif variant == "agent_defaults":
+ elif variant in {"agent_defaults", "legacy_null_wait"}:
spec = self._spec(kind="agent", config={"instructions": "Check the issue again."})
existing = create_checks_from_specs(
report=self.report, specs=[spec], attribution=ArtefactAttribution.system()
@@ -2988,7 +3086,7 @@ class TestResearchAuthoredChecks(APIBaseTest):
stored_config.update(probe_hints=[], skill_name=None)
if variant == "invalid_stored":
stored_config["retired_config_field"] = "obsolete"
- if variant in {"monitoring_disabled", "monitoring_recovered", "monitoring_visible"}:
+ if variant in {"monitoring_disabled", "monitoring_recovered", "monitoring_visible", "monitoring_null_interval"}:
stored_config["monitoring"] = {
"reference_period_minutes": 10080,
"threshold_scaling": "fixed",
@@ -2996,6 +3094,8 @@ class TestResearchAuthoredChecks(APIBaseTest):
}
if variant == "omitted_wait":
SignalReportCheck.objects.for_team(self.team.id).filter(id=existing.id).update(soak_minutes=72 * 60)
+ if variant == "legacy_null_wait":
+ SignalReportCheck.objects.for_team(self.team.id).filter(id=existing.id).update(soak_minutes=None)
if variant == "recurring":
SignalReportCheck.objects.for_team(self.team.id).filter(id=existing.id).update(
soak_minutes=1450, run_interval_minutes=7 * 24 * 60, runs_remaining=3
@@ -3019,6 +3119,14 @@ class TestResearchAuthoredChecks(APIBaseTest):
if variant == "monitoring_visible":
specs = [spec]
+ if variant == "legacy_null_wait":
+ specs = [
+ self._spec(
+ title="Revised goal", kind="agent", config=spec.config, existing_check_id=existing.id, soak_hours=1
+ )
+ ]
+ if variant == "monitoring_null_interval":
+ specs[0].config["monitoring"] = stored_config["monitoring"]
if variant == "omitted_wait":
specs[0] = self._spec(title=specs[0].title, existing_check_id=existing.id)
if variant == "invalid_stored":
@@ -3027,7 +3135,7 @@ class TestResearchAuthoredChecks(APIBaseTest):
specs[0] = specs[0].model_copy(update={"soak_hours": 72})
with patch(
"products.signals.backend.report_content_gates.feature_enabled_or_false",
- return_value=variant in {"monitoring_recovered", "monitoring_visible"},
+ return_value=variant in {"monitoring_recovered", "monitoring_visible", "monitoring_null_interval"},
):
written = create_checks_from_specs(
report=self.report,
@@ -3053,7 +3161,13 @@ class TestResearchAuthoredChecks(APIBaseTest):
if written:
assert written[0].title == specs[0].title
assert written[0].approved_at is None
- assert written[0].soak_minutes == (72 * 60 if variant == "longer_wait" else existing.soak_minutes)
+ assert written[0].soak_minutes == (
+ DEFAULT_CHECK_SOAK_HOURS * 60
+ if variant == "legacy_null_wait"
+ else 72 * 60
+ if variant == "longer_wait"
+ else existing.soak_minutes
+ )
assert written[0].run_interval_minutes == existing.run_interval_minutes
assert written[0].runs_remaining == existing.runs_remaining
assert SignalReportCheck.objects.for_team(self.team.id).filter(report=self.report).count() == (
diff --git a/products/signals/backend/test/test_research_prompt.py b/products/signals/backend/test/test_research_prompt.py
index 6fe0110b6932..4a11805665ff 100644
--- a/products/signals/backend/test/test_research_prompt.py
+++ b/products/signals/backend/test/test_research_prompt.py
@@ -299,7 +299,12 @@ class TestBuildFixVerificationPrompt:
agent_checks_enabled=agent_checks_enabled,
)
preferred = metric_checks_enabled
- for field in ("reference_period_minutes", "threshold_scaling", "baseline_period_minutes"):
+ for field in (
+ "reference_period_minutes",
+ "threshold_scaling",
+ "baseline_period_minutes",
+ "run_interval_minutes",
+ ):
assert (field in prompt) is (metric_checks_enabled and monitoring_enabled)
assert ("70-80% confidence" in prompt) is preferred
assert ('Prefer `kind: "metric_threshold"`' in prompt) is preferred