:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): keep report research alive with invalid metrics (#124565)
Mikayla Thompson37f78b0782ed
diff --git a/docs/internal/signals-pr-lifecycle.md b/docs/internal/signals-pr-lifecycle.mdindex d617a3ed8b06..3503ff037801 100644--- a/docs/internal/signals-pr-lifecycle.md+++ b/docs/internal/signals-pr-lifecycle.md@@ -64,6 +64,9 @@ The metric must match the finding's filters, population, exclusions, window, andWhen no useful supported metric can measure that claim, the summary states the specific limitation in one short sentence.Qualitative findings can omit metrics.+Research presentation drops invalid observation metrics individually and keeps the report prose and valid metrics.+A snapshot series without a paired value and measurement time is cleared while its live query remains available.+Observation metrics support HogQL filters that compare direct event fields or event properties with literal values.Boolean operations, arithmetic, and literal lists are supported; other function calls and subqueries remain unreadable.Report reads, research selection, and snapshot refresh use the same filter check.diff --git a/products/signals/backend/report_generation/research.py b/products/signals/backend/report_generation/research.pyindex bc2bc512d5fe..95355d93b05d 100644--- a/products/signals/backend/report_generation/research.py+++ b/products/signals/backend/report_generation/research.py@@ -89,11 +89,11 @@ __all__ = [def _rejection_reason(error: Exception) -> str:- """Why a chart was rejected, as failing field and rule only — never the rejected content."""+ """Return failing fields and rules without copying rejected content into logs."""if not isinstance(error, ValidationError):return type(error).__name__return ", ".join(- f"{'.'.join(str(part) for part in entry['loc']) or 'chart'}: {entry['type']}" for entry in error.errors()+ f"{'.'.join(str(part) for part in entry['loc']) or 'entry'}: {entry['type']}" for entry in error.errors())@@ -342,15 +342,22 @@ Hard rules:@field_validator("metrics", mode="before")@classmethod- def clear_legacy_metric_goals(cls, v: object) -> object:+ def drop_metrics_that_do_not_validate(cls, v: object) -> object:if not isinstance(v, list):return v- return [- {key: value for key, value in entry.items() if key not in REPORT_METRIC_GOAL_FIELDS}- if isinstance(entry, dict)- else entry- for entry in v- ]+ kept: list[ReportMetric] = []+ for index, entry in enumerate(v):+ if isinstance(entry, dict):+ entry = {key: value for key, value in entry.items() if key not in REPORT_METRIC_GOAL_FIELDS}+ try:+ kept.append(ReportMetric.model_validate(entry))+ except ValidationError as error:+ logger.warning(+ "presentation: dropped metric at index %d that did not validate (%s)",+ index,+ _rejection_reason(error),+ )+ return kept@field_validator("title", "summary")@classmethoddiff --git a/products/signals/backend/report_metrics.py b/products/signals/backend/report_metrics.pyindex 55be2b646920..c5154d48cbe1 100644--- a/products/signals/backend/report_metrics.py+++ b/products/signals/backend/report_metrics.py@@ -622,7 +622,8 @@ class ReportMetric(BaseModel):raise ValueError("value and value_at must be provided together")self._drop_a_snapshot_measured_in_the_future()if self.series is not None and self.value_at is None:- raise ValueError("series is part of the snapshot and needs value and value_at")+ logger.warning("report metric %s dropped a series without a snapshot", self.metric_id)+ self.series = Noneif self.kind == "affected_users":if self.value_format != "count":raise ValueError("an affected_users metric must use count formatting")diff --git a/products/signals/backend/test/test_report_metrics.py b/products/signals/backend/test/test_report_metrics.pyindex e8879ea720b8..7800146304fd 100644--- a/products/signals/backend/test/test_report_metrics.py+++ b/products/signals/backend/test/test_report_metrics.py@@ -4,6 +4,7 @@ import time_machinefrom django.test import SimpleTestCase+from parameterized import parameterizedfrom pydantic import ValidationErrorfrom posthog.schema import DateRange, IntervalType@@ -496,7 +497,6 @@ class TestReportMetric(SimpleTestCase):assert ReportMetric.model_validate_json(metric.model_dump_json()) == metricfor series_patch, error in (- ({"series": [1, 2], "value": None, "value_at": None}, "needs value and value_at"),({"series": [1, True]}, "not booleans"),({"series": [float("nan")]}, "finite"),({"series": [1] * (MAX_METRIC_SERIES_POINTS + 1)}, f"exceed {MAX_METRIC_SERIES_POINTS} points"),@@ -509,6 +509,23 @@ class TestReportMetric(SimpleTestCase):with self.assertRaisesRegex(ValidationError, error):ReportMetric.model_validate(content)+ @parameterized.expand([(False,), (True,)])+ def test_drops_series_without_a_snapshot_and_keeps_the_live_query(self, explicit_nulls: bool) -> None:+ content = _affected_users_metric().model_dump(mode="json")+ content["series"] = [1, 2]+ for key in ("value", "value_at"):+ if explicit_nulls:+ content[key] = None+ else:+ content.pop(key)++ metric = ReportMetric.model_validate(content)++ assert metric.series is None+ assert metric.value is None+ assert metric.value_at is None+ assert metric.query == content["query"]+def test_snapshot_timestamp_requires_a_timezone(self) -> None:content = _affected_users_metric().model_dump(mode="json")content["value_at"] = "2026-08-29T12:00:00"diff --git a/products/signals/backend/test/test_research_prompt.py b/products/signals/backend/test/test_research_prompt.pyindex 5d16bf26ef5d..daa0f35334ac 100644--- a/products/signals/backend/test/test_research_prompt.py+++ b/products/signals/backend/test/test_research_prompt.py@@ -714,6 +714,47 @@ _WINDOW_GOAL = {"goal_value": 0.01, "goal_direction": "at_most", "decision_windoclass TestReportPresentationOutputMetrics:+ @pytest.mark.parametrize("keep_valid_metric", [False, True])+ def test_invalid_metrics_preserve_prose_and_valid_metrics(+ self, keep_valid_metric: bool, caplog: pytest.LogCaptureFixture+ ) -> None:+ valid_metric = {+ "metric_id": "upload-failures",+ "title": "Upload failures",+ "kind": "occurrences",+ "role": "primary",+ "value_format": "count",+ "query": trends_metric_query(series=[{"kind": "EventsNode", "event": "upload_failed"}]),+ "series": [1, 2],+ **_WINDOW_GOAL,+ }+ with caplog.at_level(logging.WARNING):+ parsed = ReportPresentationOutput.model_validate(+ {+ "title": "fix(uploads): Handle failed uploads",+ "summary": "Uploads can fail.",+ "metrics": [+ {**valid_metric, "title": "private-metric-title", "query": {"private-filter": "private-value"}},+ None,+ *([valid_metric] if keep_valid_metric else []),+ ],+ }+ )++ assert parsed.title == "fix(uploads): Handle failed uploads"+ assert parsed.summary == "Uploads can fail."+ assert [metric.metric_id for metric in parsed.metrics] == (["upload-failures"] if keep_valid_metric else [])+ if keep_valid_metric:+ assert parsed.metrics[0].query == valid_metric["query"]+ assert parsed.metrics[0].series is None+ assert parsed.metrics[0].goal_value is None+ warning = "".join(record.getMessage() for record in caplog.records)+ assert "dropped metric at index 0" in warning+ assert "query: value_error" in warning+ assert "private-metric-title" not in warning+ assert "private-filter" not in warning+ assert "private-value" not in warning+@pytest.mark.parametrize("goal",[