:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(insights): run full data checks only on request (#121973)
Sam Penningtonb11ca07eb1be
diff --git a/products/product_analytics/backend/facade/data_checks.py b/products/product_analytics/backend/facade/data_checks.pyindex 7f1985a3490f..88144bae76e8 100644--- a/products/product_analytics/backend/facade/data_checks.py+++ b/products/product_analytics/backend/facade/data_checks.py@@ -13,15 +13,22 @@ from posthog.models.user import Userfrom products.product_analytics.backend import insight_data_checksfrom products.product_analytics.backend.facade.contracts import InsightDataChecks, TeamEmailAccess+from products.product_analytics.backend.facade.enums import InsightDataChecksDepthdef run_insight_data_checks(- query: Mapping[str, Any], *, team: Team, user: User, email_access: TeamEmailAccess+ query: Mapping[str, Any],+ *,+ team: Team,+ user: User,+ email_access: TeamEmailAccess,+ depth: InsightDataChecksDepth = InsightDataChecksDepth.QUICK,) -> InsightDataChecks:"""Check the events an insight query reads for internal users, local traffic, bots and events sent twice.The checks read the events as `user`. They do not run when a property they read is hidden from that person.The internal users check matches only the member emails and verified domains that `email_access` allows, andnames the emails it finds only when `email_access` allows person emails.+ A quick check does not run when it would read a lot of data. A full check runs with the insight query's own limits."""- return insight_data_checks.run_insight_data_checks(query, team, user, email_access=email_access)+ return insight_data_checks.run_insight_data_checks(query, team, user, email_access=email_access, depth=depth)diff --git a/products/product_analytics/backend/facade/enums.py b/products/product_analytics/backend/facade/enums.pyindex b5f195b37d35..5265438ff2de 100644--- a/products/product_analytics/backend/facade/enums.py+++ b/products/product_analytics/backend/facade/enums.py@@ -45,6 +45,7 @@ class InsightDataCheckSkipReason(LabeledStrEnum):FILTERED_BY_TRAFFIC_TYPE = "filtered_by_traffic_type", "The insight already filters on traffic type"HIDDEN_PROPERTY = "hidden_property", "A property the check reads is hidden from you"INVALID_FILTER = "invalid_filter", "The project's internal and test users filter cannot run"+ ON_REQUEST = "on_request", "The check would read a lot of data, so it runs only on request"class InsightDataCheckAction(LabeledStrEnum):@@ -52,10 +53,17 @@ class InsightDataCheckAction(LabeledStrEnum):OPEN_FILTER_SETTINGS = "open_filter_settings", "Change the project's internal and test users filter"+class InsightDataChecksDepth(LabeledStrEnum):+ QUICK = "quick", "Read within a small budget, so the checks can run on every load"+ FULL = "full", "Read with the same limits as the insight's own query"++class InsightDataChecksStatus(LabeledStrEnum):PASSED = "passed", "No data issues found"WARNING = "warning", "A check found a small issue"PROBLEM = "problem", "A check found a large issue"+ PARTIAL = "partial", "Some checks could not run, and the others found no issue"+ ON_REQUEST = "on_request", "The checks would read a lot of data, so they run only on request"NO_EVENTS = "no_events", "No events to check"UNSUPPORTED = "unsupported", "This kind of insight cannot be checked"INCONCLUSIVE = "inconclusive", "No check could run"diff --git a/products/product_analytics/backend/insight_data_check_results.py b/products/product_analytics/backend/insight_data_check_results.pyindex 6666e1d023c0..3fcd3cbdfc8f 100644--- a/products/product_analytics/backend/insight_data_check_results.py+++ b/products/product_analytics/backend/insight_data_check_results.py@@ -23,6 +23,10 @@ PROBLEM_SHARE = 0.2# An event with no user agent has the type "Automation", and server-side SDKs send those, so it is not a crawler.CRAWLER_TRAFFIC_TYPES = ("Bot", "AI Agent")+_SKIP_REASONS_WITH_NOTHING_TO_CHECK = frozenset(+ {InsightDataCheckSkipReason.FILTERED_BY_TRAFFIC_TYPE, InsightDataCheckSkipReason.TOO_FEW_SAMPLED_EVENTS}+)+LOCAL_HOST_PATTERN = r"(?i)^(localhost|[a-z0-9.-]+\.localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\])(:[0-9]+)?$"@@ -259,4 +263,9 @@ def status_of(checks: list[InsightDataCheck]) -> InsightDataChecksStatus:return InsightDataChecksStatus.PROBLEMif InsightDataCheckLevel.WARNING in levels:return InsightDataChecksStatus.WARNING+ if any(+ check.level == InsightDataCheckLevel.SKIPPED and check.skip_reason not in _SKIP_REASONS_WITH_NOTHING_TO_CHECK+ for check in checks+ ):+ return InsightDataChecksStatus.PARTIALreturn InsightDataChecksStatus.PASSEDdiff --git a/products/product_analytics/backend/insight_data_checks.py b/products/product_analytics/backend/insight_data_checks.pyindex 3435b1c99588..d306b014eeba 100644--- a/products/product_analytics/backend/insight_data_checks.py+++ b/products/product_analytics/backend/insight_data_checks.py@@ -1,20 +1,24 @@"""Checks the events an insight reads for internal users, local development, bots and events sent twice."""+import mathfrom collections.abc import Mappingfrom datetime import date, datetime, time, timedelta+from time import monotonicfrom typing import Any, castfrom zoneinfo import ZoneInfo-from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode+from posthog.schema import HogQLQueryModifiers, InlineCohortCalculation, PersonsOnEventsModefrom posthog.hogql import astfrom posthog.hogql.constants import HogQLGlobalSettingsfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.database.database import Database+from posthog.hogql.database.models import DatabaseField+from posthog.hogql.database.schema.events import EventsTablefrom posthog.hogql.errors import ExposedHogQLError, ResolutionErrorfrom posthog.hogql.parser import parse_expr, parse_selectfrom posthog.hogql.property import property_to_expr-from posthog.hogql.query import execute_hogql_query+from posthog.hogql.query import HogQLQueryExecutor, execute_hogql_queryfrom posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_exprfrom posthog.clickhouse.query_tagging import Feature, Product, tags_context@@ -43,6 +47,7 @@ from products.product_analytics.backend.facade.contracts import ()from products.product_analytics.backend.facade.enums import (InsightDataCheckKey,+ InsightDataChecksDepth,InsightDataCheckSkipReason,InsightDataChecksStatus,)@@ -80,6 +85,14 @@ MAX_EXECUTION_SECONDS = 5MAX_BYTES_READ = 10_000_000_000# The sample reads wide columns and sorts them for the repeat window, so it gets a tighter limit.MAX_SAMPLE_BYTES_READ = 1_000_000_000+MAX_ESTIMATED_BYTES = 10_000_000_000+# Larger than the JSON of almost any event. Below it, the checks skip the size read, which can read hundreds of MB.+# An event above it only makes a check stop at MAX_BYTES_READ, which offers the person the same button.+MAX_JSON_BYTES_PER_EVENT = 10_000+# A full check has the same time limit as the insight's own query, and no read limit. Its queries share the limit,+# so the whole run answers before the page and the MCP client stop waiting at 120 seconds.+FULL_MAX_EXECUTION_SECONDS = 60+JSON_SIZE_SAMPLE = 1_000# A person cannot repeat an action this fast, but an app that sends an event twice does.REPEAT_WINDOW_MILLISECONDS = 100@@ -198,6 +211,38 @@ _SAMPLE_COUNTS_QUERY = """)"""+_JSON_SIZE_QUERY = """+ SELECT avg(json_bytes)+ FROM {sampled_events}+"""++_JSON_COLUMN_FIELDS = {"properties": ("properties",), "person_properties": ("person", "properties")}+# A property group map holds part of a JSON column, so the size of that JSON column bounds what reading the map costs.+_PROPERTY_MAP_PREFIXES = {"properties_group_": "properties", "person_properties_map_": "person_properties"}+++@frozen+class _Limits:+ max_execution_seconds: int+ max_bytes_read: int | None+ max_sample_bytes_read: int | None+ deadline: float | None = None+++def _limits(depth: InsightDataChecksDepth) -> _Limits:+ if depth == InsightDataChecksDepth.FULL:+ return _Limits(+ max_execution_seconds=FULL_MAX_EXECUTION_SECONDS,+ max_bytes_read=None,+ max_sample_bytes_read=None,+ deadline=monotonic() + FULL_MAX_EXECUTION_SECONDS,+ )+ return _Limits(+ max_execution_seconds=MAX_EXECUTION_SECONDS,+ max_bytes_read=MAX_BYTES_READ,+ max_sample_bytes_read=MAX_SAMPLE_BYTES_READ,+ )+@frozenclass _EventReader:@@ -206,10 +251,19 @@ class _EventReader:modifiers: HogQLQueryModifiers# The database the chart's events were read with, so the checks do not build another.database: Database+ limits: _Limits++ def context(self) -> HogQLContext:+ return HogQLContext(team_id=self.team.pk, user=self.user, database=self.database)def rows_within_limits(- self, query: ast.SelectQuery, *, query_type: str, max_bytes_read: int+ self, query: ast.SelectQuery, *, query_type: str, max_bytes_read: int | None, max_threads: int | None = None) -> list[list[Any]] | Skipped:+ max_execution_seconds = self.limits.max_execution_seconds+ if self.limits.deadline is not None:+ max_execution_seconds = min(max_execution_seconds, math.floor(self.limits.deadline - monotonic()))+ if max_execution_seconds < 1:+ return Skipped(reason=InsightDataCheckSkipReason.TIME_LIMIT)try:response = execute_hogql_query(query=query,@@ -217,8 +271,8 @@ class _EventReader:user=self.user,query_type=query_type,modifiers=self.modifiers,- context=HogQLContext(team_id=self.team.pk, user=self.user, database=self.database),- settings=_query_settings(max_bytes_read),+ context=self.context(),+ settings=_query_settings(max_bytes_read, max_execution_seconds, max_threads),)except CHQueryErrorTooManyBytes:return Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT)@@ -229,13 +283,16 @@ class _EventReader:return [list(row) for row in response.results or []]-def _query_settings(max_bytes_read: int) -> HogQLGlobalSettings:+def _query_settings(+ max_bytes_read: int | None, max_execution_seconds: int, max_threads: int | None+) -> HogQLGlobalSettings:# Without "throw", a query that reaches a limit can return the events it counted so far as the answer.return HogQLGlobalSettings(- max_execution_time=MAX_EXECUTION_SECONDS,+ max_execution_time=max_execution_seconds,timeout_overflow_mode="throw",max_bytes_to_read=max_bytes_read,- read_overflow_mode="throw",+ read_overflow_mode="throw" if max_bytes_read is not None else None,+ max_threads=max_threads,)@@ -262,7 +319,12 @@ def _check_modifiers(chart_modifiers: HogQLQueryModifiers, *, keep_project_bot_r# With `debug` on, a failed query returns an error and no rows, which would read as no events.# Without the pushdown, a persons join reads the latest version of every person in the project, however few# events the check reads.- update: dict[str, Any] = {"debug": False, "personIdPushdown": True}+ # An inlined cohort scans every person in the project, so a cohort in a filter reads its stored members instead.+ update: dict[str, Any] = {+ "debug": False,+ "personIdPushdown": True,+ "inlineCohortCalculation": InlineCohortCalculation.OFF,+ }if not keep_project_bot_rules:# The bot check uses PostHog's definition of a bot, so a project's own bot rules do not apply.update["customBotDefinitions"] = []@@ -326,7 +388,9 @@ def _count_events_per_day(where=ast.And(exprs=[events.sort_key_where, _between_days(events, days[0], days[-1], timezone)]),sample=_sample_of(1, SIZING_SAMPLE_ONE_IN),)- rows = reader.rows_within_limits(query, query_type="insight_data_checks_sizing", max_bytes_read=MAX_BYTES_READ)+ rows = reader.rows_within_limits(+ query, query_type="insight_data_checks_sizing", max_bytes_read=reader.limits.max_bytes_read+ )if isinstance(rows, Skipped):return rows@@ -508,9 +572,9 @@ def _internal_users_match()-def _count_every_event(- events: ChartEvents, reader: _EventReader, where: ast.Expr, internal_users: _InternalUsersMatch | Skipped-) -> CountedEvents | Skipped:+def _counted_events_query(+ events: ChartEvents, where: ast.Expr, internal_users: _InternalUsersMatch | Skipped+) -> ast.SelectQuery:match = _NO_INTERNAL_USERS if isinstance(internal_users, Skipped) else internal_usersquery = events.select(_COUNTED_EVENTS_QUERY,@@ -522,7 +586,16 @@ def _count_every_event()if match.person_fields is not None:cast(ast.JoinExpr, query.select_from).next_join = _current_person_join(events, where, match.person_fields)- rows = reader.rows_within_limits(query, query_type="insight_data_checks_counted", max_bytes_read=MAX_BYTES_READ)+ return query+++def _count_every_event(+ events: ChartEvents, reader: _EventReader, where: ast.Expr, internal_users: _InternalUsersMatch | Skipped+) -> CountedEvents | Skipped:+ query = _counted_events_query(events, where, internal_users)+ rows = reader.rows_within_limits(+ query, query_type="insight_data_checks_counted", max_bytes_read=reader.limits.max_bytes_read+ )if isinstance(rows, Skipped):return rows@@ -560,7 +633,9 @@ def _count_local_events_the_filter_keeps(if person_fields is not None:local_events = ast.And(exprs=[clone_expr(where), clone_expr(is_local_host)])cast(ast.JoinExpr, query.select_from).next_join = _current_person_join(events, local_events, person_fields)- rows = reader.rows_within_limits(query, query_type="insight_data_checks_local", max_bytes_read=MAX_BYTES_READ)+ rows = reader.rows_within_limits(+ query, query_type="insight_data_checks_local", max_bytes_read=reader.limits.max_bytes_read+ )if isinstance(rows, Skipped):return rows@@ -624,7 +699,7 @@ def _sample_events(rows = reader.rows_within_limits(cast(ast.SelectQuery, query),query_type="insight_data_checks_sample",- max_bytes_read=MAX_SAMPLE_BYTES_READ,+ max_bytes_read=reader.limits.max_sample_bytes_read,)if isinstance(rows, Skipped):return rows@@ -644,6 +719,12 @@ def _sample_events()+def _checked_where(events: ChartEvents, window: CheckWindow, timezone: ZoneInfo) -> ast.Expr:+ # The bounds apply to a whole range too, because a lifecycle chart also reads the interval before its range.+ first, last = window.days_to_check[0], window.days_to_check[-1]+ return ast.And(exprs=[events.where, _between_days(events, first, last, timezone)])++def _read_checked_days(events: ChartEvents,reader: _EventReader,@@ -654,9 +735,7 @@ def _read_checked_days(filters_traffic_type: bool,email_access: TeamEmailAccess,) -> CheckedEvents:- # The bounds apply to a whole range too, because a lifecycle chart also reads the interval before its range.- first, last = window.days_to_check[0], window.days_to_check[-1]- where = ast.And(exprs=[events.where, _between_days(events, first, last, timezone)])+ where = _checked_where(events, window, timezone)counted: CountedEvents | Skipped = Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS)if window.events_on_days_to_check <= MAX_COUNTED_EVENTS:@@ -668,13 +747,130 @@ def _read_checked_days(return CheckedEvents(counted=counted, local=local, sample=sample, insight_filters_traffic_type=filters_traffic_type)+def _reads_the_events_table(field_type: ast.FieldType) -> bool:+ table_type: ast.Type | None = field_type.table_type+ while isinstance(table_type, ast.TableAliasType | ast.ColumnAliasedTableType | ast.VirtualTableType):+ table_type = table_type.table_type+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)+++class _EventJsonReads(TraversingVisitor):+ """The JSON columns of the events table that a printed query reads a property from."""++ def __init__(self, context: HogQLContext) -> None:+ super().__init__()+ self.context = context+ self.columns: set[str] = set()++ # A property read and a key-existence test both reach the JSON column through a field, so the field is enough.+ def visit_field(self, node: ast.Field) -> None:+ name = str(node.chain[-1])+ self.columns.update(column for prefix, column in _PROPERTY_MAP_PREFIXES.items() if name.startswith(prefix))+ field_type = node.type+ if (+ isinstance(field_type, ast.FieldType)+ and field_type.name in _JSON_COLUMN_FIELDS+ and _reads_the_events_table(field_type)+ ):+ column = field_type.resolve_database_field(self.context)+ if isinstance(column, DatabaseField) and column.name in _JSON_COLUMN_FIELDS:+ self.columns.add(column.name)+ super().visit_field(node)++ # A native JSON column stores each path as a subcolumn, so this read is narrow, however wide the document.+ def visit_json_subcolumn_access(self, node: ast.JsonSubcolumnAccess) -> None:+ return+++def _json_columns_read(query: ast.SelectQuery, reader: _EventReader) -> frozenset[str]:+ """The JSON columns `query` reads a property from, decided as the query is printed to run."""+ executor = HogQLQueryExecutor(+ query=clone_expr(query),+ team=reader.team,+ user=reader.user,+ modifiers=reader.modifiers,+ context=reader.context(),+ )+ executor.generate_clickhouse_sql()+ if executor.clickhouse_prepared_ast is None or executor.clickhouse_context is None:+ return frozenset()+ reads = _EventJsonReads(executor.clickhouse_context)+ reads.visit(executor.clickhouse_prepared_ast)+ return frozenset(reads.columns)+++def _json_bytes_per_event(+ events: ChartEvents, reader: _EventReader, where: ast.Expr, columns: frozenset[str]+) -> float | Skipped:+ """The average size of the JSON columns `columns` across a few of the events `where` keeps."""+ json_bytes = parse_expr(+ " + ".join(f"length({{{column}}})" for column in sorted(columns)),+ placeholders={column: ast.Field(chain=list(_JSON_COLUMN_FIELDS[column])) for column in columns},+ )+ sampled_events = events.select("SELECT {json_bytes} AS json_bytes FROM events", where=where, json_bytes=json_bytes)+ sampled_events.limit = ast.Constant(value=JSON_SIZE_SAMPLE)+ query = parse_select(_JSON_SIZE_QUERY, placeholders={"sampled_events": sampled_events})+ rows = reader.rows_within_limits(+ cast(ast.SelectQuery, query),+ query_type="insight_data_checks_json_size",+ max_bytes_read=reader.limits.max_bytes_read,+ # Each thread reads at least one block of the JSON column before the limit stops it, so one thread reads least.+ max_threads=1,+ )+ if isinstance(rows, Skipped):+ return rows+ # The average of no events is NaN, and no events cost nothing to read.+ average = float(rows[0][0] or 0) if rows else 0.0+ return 0.0 if math.isnan(average) else average+++def _would_read_too_much(+ events: ChartEvents,+ reader: _EventReader,+ window: CheckWindow,+ timezone: ZoneInfo,+ *,+ email_access: TeamEmailAccess,+ hidden_properties: RestrictedPropertyNames,+) -> bool:+ """Whether the count of every event, which is the most any check reads, would read more than MAX_ESTIMATED_BYTES.++ Only a property without a column of its own makes a check heavy, because it is read from a JSON column of every+ event the check reads.+ """+ if window.events_on_days_to_check * MAX_JSON_BYTES_PER_EVENT <= MAX_ESTIMATED_BYTES:+ return False++ where = _checked_where(events, window, timezone)+ try:+ internal_users = _internal_users_match(+ events, reader, email_access=email_access, hidden_properties=hidden_properties+ )+ columns = _json_columns_read(_counted_events_query(events, where, internal_users), reader)+ except (ExposedHogQLError, ResolutionError):+ # The count leaves out a project filter that cannot run, so the estimate leaves it out too.+ no_filter = Skipped(reason=InsightDataCheckSkipReason.INVALID_FILTER)+ columns = _json_columns_read(_counted_events_query(events, where, no_filter), reader)+ if not columns:+ return False++ first, last = window.days_to_check[0], window.days_to_check[-1]+ # The count reads each event of the chart's event names and days before its filters drop any.+ sort_key_where = ast.And(exprs=[events.sort_key_where, _between_days(events, first, last, timezone)])+ bytes_per_event = _json_bytes_per_event(events, reader, sort_key_where, columns)+ # A size read that stops at a limit means the full count would stop there too.+ if isinstance(bytes_per_event, Skipped):+ return True+ return window.events_on_days_to_check * bytes_per_event > MAX_ESTIMATED_BYTES++def _sample_summary(sample: SampledEvents | Skipped) -> InsightDataChecksSample | None:if isinstance(sample, Skipped):return Nonereturn InsightDataChecksSample(rate=sample.rate, events=sample.events, distinct_ids=sample.distinct_ids)-def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents) -> InsightDataChecksCoverage:+def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents | None) -> InsightDataChecksCoverage:return InsightDataChecksCoverage(date_from=window.days_in_range[0],date_to=window.days_in_range[-1],@@ -683,8 +879,8 @@ def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents)checked_date_to=window.days_to_check[-1],days_checked=len(window.days_to_check),filters_test_accounts=events.filters_test_accounts,- events_checked=events_on_checked_days(checked),- sample=_sample_summary(checked.sample),+ events_checked=events_on_checked_days(checked) if checked else None,+ sample=_sample_summary(checked.sample) if checked else None,)@@ -698,7 +894,12 @@ def _nothing_checked(def run_insight_data_checks(- query: Mapping[str, Any], team: Team, user: User, *, email_access: TeamEmailAccess+ query: Mapping[str, Any],+ team: Team,+ user: User,+ *,+ email_access: TeamEmailAccess,+ depth: InsightDataChecksDepth = InsightDataChecksDepth.QUICK,) -> InsightDataChecks:with tags_context(product=Product.PRODUCT_ANALYTICS, feature=Feature.INSIGHT, name="insight_data_checks"):events = chart_events(query, team, user)@@ -715,11 +916,20 @@ def run_insight_data_checks(user=user,modifiers=_check_modifiers(events.modifiers, keep_project_bot_rules=filters_traffic_type),database=events.database,+ limits=_limits(depth),)timezone = ZoneInfo(team.timezone)window = _choose_window(events, reader, timezone)if isinstance(window, Skipped):return _nothing_checked(InsightDataChecksStatus.INCONCLUSIVE, reason=window.reason)+ if depth == InsightDataChecksDepth.QUICK and _would_read_too_much(+ events, reader, window, timezone, email_access=email_access, hidden_properties=hidden_properties+ ):+ return _nothing_checked(+ InsightDataChecksStatus.ON_REQUEST,+ _coverage(events, window, None),+ InsightDataCheckSkipReason.ON_REQUEST,+ )checked = _read_checked_days(events,reader,diff --git a/products/product_analytics/backend/presentation/insight.py b/products/product_analytics/backend/presentation/insight.pyindex 1e44cd43e14c..59d1950ef88b 100644--- a/products/product_analytics/backend/presentation/insight.py+++ b/products/product_analytics/backend/presentation/insight.py@@ -187,6 +187,7 @@ from products.product_analytics.backend.facade.enums import (InsightDataCheckKey,InsightDataCheckLevel,InsightDataCheckMethod,+ InsightDataChecksDepth,InsightDataCheckSkipReason,InsightDataChecksStatus,)@@ -1698,6 +1699,13 @@ class InsightBulkSetTestAccountFilterResponseSerializer(serializers.Serializer):)+DEPTH_HELP_TEXT = (+ "`quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would "+ "read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the "+ "same limits as the insight's own query."+)++class InsightDataChecksRequestSerializer(serializers.Serializer):query = QueryFieldSerializer(help_text=(@@ -1713,6 +1721,21 @@ class InsightDataChecksRequestSerializer(serializers.Serializer):"`query/{client_query_id}/`."),)+ depth = serializers.ChoiceField(+ choices=InsightDataChecksDepth.choices,+ required=False,+ default=InsightDataChecksDepth.QUICK,+ help_text=DEPTH_HELP_TEXT,+ )+++class InsightDataChecksQueryParamsSerializer(serializers.Serializer):+ depth = serializers.ChoiceField(+ choices=InsightDataChecksDepth.choices,+ required=False,+ default=InsightDataChecksDepth.QUICK,+ help_text=DEPTH_HELP_TEXT,+ )class InsightDataCheckFilterSerializer(serializers.Serializer):@@ -1876,7 +1899,9 @@ class InsightDataChecksSerializer(serializers.Serializer):help_text=("The worst result across the checks. `no_events` when no event matches the insight's events, filters ""and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, "- "and when a property the checks read is hidden from the caller. `inconclusive` when no check could run."+ "and when a property the checks read is hidden from the caller. `inconclusive` when no check could run. "+ "`partial` when some checks could not run and the others found nothing. `on_request` when the checks "+ "would read a lot of data for a quick check, so they did not run: send `depth` `full` to run them."),)checks = InsightDataCheckSerializer(many=True, help_text="Every check, in a fixed order, whatever its result.")@@ -2654,9 +2679,10 @@ When set, the specified dashboard's filters and date range override will be applraise PermissionDenied("You need query access to check a query that is not saved as an insight.")if client_query_id := request.validated_data.get("client_query_id"):tag_queries(client_query_id=client_query_id)- return self._data_checks_response(request.validated_data["query"])+ return self._data_checks_response(request.validated_data["query"], depth=request.validated_data["depth"])@validated_request(+ query_serializer=InsightDataChecksQueryParamsSerializer,parameters=[INSIGHT_ID_PATH_PARAMETER],responses={200: OpenApiResponse(response=InsightDataChecksSerializer, description="The result of every check.")@@ -2671,9 +2697,12 @@ When set, the specified dashboard's filters and date range override will be appl@action(methods=["GET"], detail=True, url_path="data_checks", required_scopes=["insight:read"])def saved_insight_data_checks(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response:insight = self.get_object()- return self._data_checks_response(upgrade(insight.query) if insight.query else {})+ return self._data_checks_response(+ upgrade(insight.query) if insight.query else {},+ depth=request.validated_query_data["depth"],+ )- def _data_checks_response(self, query: dict[str, Any]) -> Response:+ def _data_checks_response(self, query: dict[str, Any], *, depth: InsightDataChecksDepth) -> Response:# A personal API key reads within the same hourly budget as the query API, and each check query debits it.if get_query_tag_value("access_method") == AccessMethod.PERSONAL_API_KEY:tag_queries(api_queries_budgeted=True)@@ -2685,6 +2714,7 @@ When set, the specified dashboard's filters and date range override will be applteam=self.team,user=cast(User, self.request.user),email_access=self._team_email_access(),+ depth=InsightDataChecksDepth(depth),)except CHQueryErrorQueryWasCancelled:# The client cancelled the run, so nothing reads a body. 499 is Nginx's "Client Closed Request".diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.pyindex bd496806a12d..e1b84bd77408 100644--- a/products/product_analytics/backend/tests/test_insight_data_checks.py+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py@@ -6,9 +6,17 @@ from types import MappingProxyTypefrom typing import Anyimport time_machine-from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events+from posthog.test.base import (+ APIBaseTest,+ ClickhouseTestMixin,+ _create_event,+ _create_person,+ cleanup_materialized_columns,+ flush_persons_and_events,+)from unittest.mock import patch+from django.conf import settingsfrom django.test import SimpleTestCasefrom parameterized import parameterized@@ -37,6 +45,7 @@ from products.product_analytics.backend.facade.data_checks import run_insight_dafrom products.product_analytics.backend.facade.enums import (InsightDataCheckKey,InsightDataCheckLevel,+ InsightDataChecksDepth,InsightDataCheckSkipReason,)from products.product_analytics.backend.facade.models import Insight@@ -53,6 +62,8 @@ from products.product_analytics.backend.insight_data_check_results import ()from products.product_analytics.backend.insight_data_check_team_emails import TeamEmails, team_email_pattern+from ee.clickhouse.materialized_columns.columns import materialize+PASSED = InsightDataCheckLevel.PASSEDWARNING = InsightDataCheckLevel.WARNINGPROBLEM = InsightDataCheckLevel.PROBLEM@@ -78,6 +89,7 @@ FROM_A_BOT = {"key": "$raw_user_agent", "value": [BOT_USER_AGENT], "operator": "NOT_A_BOT = {"key": "$virt_traffic_type", "value": ["Bot"], "operator": "is_not", "type": "event"}IS_PAID = {"key": "plan", "value": ["paid"], "operator": "exact", "type": "event"}HAS_NO_PLAN_TODAY = {"key": "plan", "value": "is_not_set", "operator": "is_not_set", "type": "person"}+SENT_WITHOUT_A_PLAN = {"key": "plan", "value": "is_not_set", "operator": "is_not_set", "type": "event"}ONE_STEP = [{"kind": "EventsNode", "event": EVENT}]TWO_STEPS = [{"kind": "EventsNode", "event": EVENT}, {"kind": "EventsNode", "event": EVENT}]SEVEN_DAYS = [date(2026, 3, 1) + timedelta(days=offset) for offset in range(7)]@@ -215,16 +227,33 @@ class TestEvaluateDataChecks(SimpleTestCase):internal_users = next(check for check in checks if check.key == INTERNAL)assert (internal_users.events, internal_users.action) == (removed_by_filter + missed_by_filter, action)- def test_status_is_inconclusive_when_every_check_is_skipped(self) -> None:- checks = evaluate_data_checks(- CheckedEvents(- counted=Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS),- local=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),- sample=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),- )- )-- assert status_of(checks) == "inconclusive"+ @parameterized.expand(+ [+ (+ "every_check_skipped",+ CheckedEvents(+ counted=Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS),+ local=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),+ sample=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),+ ),+ "inconclusive",+ ),+ (+ "sample_hit_the_read_limit",+ replace(checked(1000), sample=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT)),+ "partial",+ ),+ (+ "bot_check_not_needed_because_the_insight_filters_on_traffic_type",+ replace(checked(1000), insight_filters_traffic_type=True),+ "passed",+ ),+ ]+ )+ def test_status_says_whether_every_check_ran(+ self, _name: str, checked_events: CheckedEvents, expected: str+ ) -> None:+ assert status_of(evaluate_data_checks(checked_events)) == expectedclass TestNewestDaysWithin(SimpleTestCase):@@ -324,12 +353,15 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):},}- def _check(self, query: dict[str, Any]) -> InsightDataChecks:+ def _check(+ self, query: dict[str, Any], depth: InsightDataChecksDepth = InsightDataChecksDepth.QUICK+ ) -> InsightDataChecks:return run_insight_data_checks(query,team=self.team,user=self.user,email_access=TeamEmailAccess(member_emails=True, verified_domains=True, person_emails=True),+ depth=depth,)def _internal_users(self, result: InsightDataChecks) -> InsightDataCheck:@@ -749,19 +781,73 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):@parameterized.expand([- ("unknown_event", [{"kind": "EventsNode", "event": "never sent"}], 1_000_000_000, "no_events"),- ("read_limit_reached_while_sizing", ONE_STEP, 1, "inconclusive"),+ ("unknown_event", [{"kind": "EventsNode", "event": "never sent"}], 1_000_000_000, "quick", "no_events"),+ ("read_limit_reached_while_sizing", ONE_STEP, 1, "quick", "inconclusive"),+ ("full_check_past_its_deadline", ONE_STEP, 1_000_000_000, "full", "inconclusive"),])+ @patch(f"{CHECKS_MODULE}.FULL_MAX_EXECUTION_SECONDS", 0)def test_reports_why_nothing_was_checked(- self, _name: str, series: list[dict[str, Any]], max_bytes_read: int, expected: str+ self, _name: str, series: list[dict[str, Any]], max_bytes_read: int, depth: str, expected: str) -> None:with patch(f"{CHECKS_MODULE}.MAX_BYTES_READ", max_bytes_read):- result = self._check(self._query(series))+ result = self._check(self._query(series), InsightDataChecksDepth(depth))assert result.status == expectedassert {check.level for check in result.checks} == {"skipped"}+ # The native events table reads a property from a subcolumn of its own, so a property filter reads little there.+ @parameterized.expand(+ [+ ("properties_with_columns_of_their_own", [], None, 10_000, ("warning", "warning")),+ (+ "a_filter_on_a_property_without_a_column",+ [SENT_WITHOUT_A_PLAN],+ None,+ 10_000,+ ("on_request", "warning"),+ ),+ (+ "too_few_events_to_read_a_lot_even_from_large_events",+ [SENT_WITHOUT_A_PLAN],+ None,+ 0,+ ("warning", "warning"),+ ),+ ("asked_for_in_the_request_body", [SENT_WITHOUT_A_PLAN], "body", 10_000, ("warning", "warning")),+ ("asked_for_on_a_saved_insight", [SENT_WITHOUT_A_PLAN], "saved_insight", 10_000, ("warning", "warning")),+ ]+ )+ @patch(f"{CHECKS_MODULE}.MAX_ESTIMATED_BYTES", 1)+ def test_runs_the_checks_that_would_read_a_lot_only_on_request(+ self,+ _name: str,+ properties: list[dict[str, Any]],+ asked_for: str | None,+ max_json_bytes_per_event: int,+ status_by_events_schema: tuple[str, str],+ ) -> None:+ legacy_status, native_status = status_by_events_schema+ expected_status = native_status if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else legacy_status+ self.enterContext(patch(f"{CHECKS_MODULE}.MAX_JSON_BYTES_PER_EVENT", max_json_bytes_per_event))+ self.addCleanup(cleanup_materialized_columns)+ materialize("events", "email", table_column="person_properties")+ # A lifecycle chart filters on this property, which production reads from a column of its own.+ materialize("events", "$process_person_profile")+ query = self._query(+ properties=properties, modifiers={"personsOnEventsMode": "person_id_override_properties_on_events"}+ )++ if asked_for == "saved_insight":+ insight = Insight.objects.create(team=self.team, query=query)+ response = self.client.get(f"/api/projects/{self.team.id}/insights/{insight.id}/data_checks/?depth=full")+ else:+ body = {"query": query, "depth": "full"} if asked_for == "body" else {"query": query}+ response = self.client.post(f"/api/projects/{self.team.id}/insights/data_checks/", body, format="json")++ assert response.status_code == status.HTTP_200_OK, response.content+ assert response.json()["status"] == expected_status+def test_a_filter_that_cannot_run_raises_with_the_debug_modifier_on(self) -> None:query = self._query(properties=[{"type": "hogql", "key": "throwIf(1, 'stop') = 0"}], modifiers={"debug": True})diff --git a/products/product_analytics/frontend/generated/api.schemas.ts b/products/product_analytics/frontend/generated/api.schemas.tsindex 1038a9caa8ad..a252e090a59b 100644--- a/products/product_analytics/frontend/generated/api.schemas.ts+++ b/products/product_analytics/frontend/generated/api.schemas.ts@@ -9072,6 +9072,8 @@ export interface ActivityLogPaginatedResponseApi {* * `passed` - No data issues found* * `warning` - A check found a small issue* * `problem` - A check found a large issue+ * * `partial` - Some checks could not run, and the others found no issue+ * * `on_request` - The checks would read a lot of data, so they run only on request* * `no_events` - No events to check* * `unsupported` - This kind of insight cannot be checked* * `inconclusive` - No check could run@@ -9083,6 +9085,8 @@ export const InsightDataChecksStatusEnumApi = {Passed: 'passed',Warning: 'warning',Problem: 'problem',+ Partial: 'partial',+ OnRequest: 'on_request',NoEvents: 'no_events',Unsupported: 'unsupported',Inconclusive: 'inconclusive',@@ -9128,6 +9132,7 @@ export const InsightDataCheckLevelEnumApi = {* * `filtered_by_traffic_type` - The insight already filters on traffic type* * `hidden_property` - A property the check reads is hidden from you* * `invalid_filter` - The project's internal and test users filter cannot run+ * * `on_request` - The check would read a lot of data, so it runs only on request*/export type InsightDataCheckSkipReasonEnumApi =(typeof InsightDataCheckSkipReasonEnumApi)[keyof typeof InsightDataCheckSkipReasonEnumApi]@@ -9141,6 +9146,7 @@ export const InsightDataCheckSkipReasonEnumApi = {FilteredByTrafficType: 'filtered_by_traffic_type',HiddenProperty: 'hidden_property',InvalidFilter: 'invalid_filter',+ OnRequest: 'on_request',} as const/**@@ -9202,7 +9208,8 @@ export interface InsightDataCheckApi {* * `too_few_sampled_events` - The sample has too few events to give a share* * `filtered_by_traffic_type` - The insight already filters on traffic type* * `hidden_property` - A property the check reads is hidden from you- * * `invalid_filter` - The project's internal and test users filter cannot run */+ * * `invalid_filter` - The project's internal and test users filter cannot run+ * * `on_request` - The check would read a lot of data, so it runs only on request */skip_reason: InsightDataCheckSkipReasonEnumApi | null/** `counted` when the check counted every event on the days checked, `sampled` when it read a sample. Null when the check did not run.*@@ -9297,11 +9304,13 @@ export interface InsightDataChecksCoverageApi {}export interface InsightDataChecksApi {- /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run.+ /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run. `partial` when some checks could not run and the others found nothing. `on_request` when the checks would read a lot of data for a quick check, so they did not run: send `depth` `full` to run them.** * `passed` - No data issues found* * `warning` - A check found a small issue* * `problem` - A check found a large issue+ * * `partial` - Some checks could not run, and the others found no issue+ * * `on_request` - The checks would read a lot of data, so they run only on request* * `no_events` - No events to check* * `unsupported` - This kind of insight cannot be checked* * `inconclusive` - No check could run */@@ -9418,6 +9427,18 @@ export interface BulkUpdateTagsResponseApi {skipped: BulkUpdateTagsErrorApi[]}+/**+ * * `quick` - Read within a small budget, so the checks can run on every load+ * * `full` - Read with the same limits as the insight's own query+ */+export type InsightDataChecksDepthEnumApi =+ (typeof InsightDataChecksDepthEnumApi)[keyof typeof InsightDataChecksDepthEnumApi]++export const InsightDataChecksDepthEnumApi = {+ Quick: 'quick',+ Full: 'full',+} as const+export interface InsightDataChecksRequestApi {/** The insight query to check, in the shape an insight stores it: an `InsightVizNode` with its `source`. Pass the query of a saved insight, or one that is not saved yet. */query: _InsightQuerySchemaApi@@ -9426,6 +9447,11 @@ export interface InsightDataChecksRequestApi {* @maxLength 128*/client_query_id?: string+ /** `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.+ *+ * * `quick` - Read within a small budget, so the checks can run on every load+ * * `full` - Read with the same limits as the insight's own query */+ depth?: InsightDataChecksDepthEnumApi}/**@@ -9861,9 +9887,25 @@ export const InsightsActivityRetrieveFormat = {} as constexport type InsightsDataChecksRetrieveParams = {+ /**+ * `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.+ *+ * * `quick` - Read within a small budget, so the checks can run on every load+ * * `full` - Read with the same limits as the insight's own query+ * @minLength 1+ */+ depth?: InsightsDataChecksRetrieveDepthformat?: InsightsDataChecksRetrieveFormat}+export type InsightsDataChecksRetrieveDepth =+ (typeof InsightsDataChecksRetrieveDepth)[keyof typeof InsightsDataChecksRetrieveDepth]++export const InsightsDataChecksRetrieveDepth = {+ Quick: 'quick',+ Full: 'full',+} as const+export type InsightsDataChecksRetrieveFormat =(typeof InsightsDataChecksRetrieveFormat)[keyof typeof InsightsDataChecksRetrieveFormat]diff --git a/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsx b/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsxindex 6252c415a2de..3d1fdb3133b8 100644--- a/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsx+++ b/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsx@@ -1,7 +1,17 @@import { useActions, useMountedLogic, useValues } from 'kea'import { useId } from 'react'-import { Accordion, Button, Popover, PopoverContent, PopoverTrigger, Skeleton, Spinner, Text } from '@posthog/quill'+import {+ Accordion,+ Button,+ Dot,+ Popover,+ PopoverContent,+ PopoverTrigger,+ Skeleton,+ Spinner,+ Text,+} from '@posthog/quill'import { humanFriendlyNumber } from 'lib/utils/numbers'import { pluralize } from 'lib/utils/strings'@@ -25,6 +35,9 @@ function summary(status: InsightDataChecksStatus, coverage: InsightDataChecksCovif (status === 'failed') {return "The checks didn't finish. The chart's own numbers are not affected."}+ if (status === 'on_request') {+ return "Checking this chart's events reads a lot of data, so the checks don't run on their own."+ }if (!coverage) {return 'No check could run on this chart.'}@@ -46,8 +59,8 @@ export interface DataChecksPopoverProps {export function DataChecksPopover({ insightProps, trigger, children }: DataChecksPopoverProps): JSX.Element {const logic = insightDataChecksLogic(insightProps)- const { visibleChecks, checkedQuery, result, status } = useValues(logic)- const { loadDataChecks, reportButtonClicked, reportDataChecksViewed } = useActions(logic)+ const { visibleChecks, checkedQuery, result, status, canRunFullCheck } = useValues(logic)+ const { loadDataChecks, reportButtonClicked, reportDataChecksViewed, runFullCheck } = useActions(logic)const { user } = useValues(userLogic)const userId = user?.uuid ?? ''// Mounted outside the popover content, so a half-written note survives the popover closing.@@ -72,29 +85,36 @@ export function DataChecksPopover({ insightProps, trigger, children }: DataCheckdata-attr="insight-data-checks-list"><Text id={titleId} size="sm" weight="medium">- Checks we've run on this data+ {status === 'on_request' ? 'Checks we can run on this data' : "Checks we've run on this data"}</Text>- {visibleChecks.length > 0 ? (+ {/* Every check waits for the same request, so the reason is said once, below the list. */}+ {visibleChecks.length > 0 && status !== 'on_request' ? (<Accordion>{visibleChecks.map((check) => (<InsightDataCheckRow key={check.key} insightProps={insightProps} check={check} />))}</Accordion>) : (- status === 'checking' && (+ (status === 'checking' || status === 'on_request') && (<ul className="-mx-2 my-0 flex list-none flex-col p-0">{Object.entries(DATA_CHECK_NAMES).map(([key, name]) => (<li key={key} className="flex items-start gap-2 px-2 py-1"><span className="flex h-5 w-4 shrink-0 items-center justify-center">- <Spinner aria-hidden className="text-(--muted-foreground)" />+ {status === 'checking' ? (+ <Spinner aria-hidden className="text-(--muted-foreground)" />+ ) : (+ <Dot variant="default" />+ )}</span><span className="flex flex-col"><Text size={null} weight="medium" render={<span />}>{name}</Text>- <span className="flex h-4 items-center">- <Skeleton className="h-2.5 w-28" />- </span>+ {status === 'checking' && (+ <span className="flex h-4 items-center">+ <Skeleton className="h-2.5 w-28" />+ </span>+ )}</span></li>))}@@ -120,7 +140,18 @@ export function DataChecksPopover({ insightProps, trigger, children }: DataCheckTry again</Button>)}- {(status === 'passed' || status === 'warning' || status === 'problem') && (+ {canRunFullCheck && (+ <Button+ variant="outline"+ size="sm"+ className="self-start"+ onClick={() => checkedQuery && runFullCheck(checkedQuery)}+ data-attr="insight-data-checks-run-full"+ >+ Run a full check+ </Button>+ )}+ {(status === 'passed' || status === 'warning' || status === 'problem' || status === 'partial') && (<DataChecksFeedback userId={userId} />)}</PopoverContent>diff --git a/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx b/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsxindex 7e8dded9d655..3a84382b7260 100644--- a/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx+++ b/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx@@ -27,7 +27,10 @@ export function DataChecksTag({<IconCheck key={status} />) : status === 'checking' ? (<Spinner key={status} textColored />- ) : status === 'no_events' || status === 'inconclusive' ? (+ ) : status === 'no_events' ||+ status === 'inconclusive' ||+ status === 'on_request' ||+ status === 'partial' ? (<IconCircleDashed key={status} />) : (<IconWarning key={status} />diff --git a/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx b/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsxindex f98b8dc94080..f21d32219489 100644--- a/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx+++ b/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx@@ -19,6 +19,10 @@ function statusLabel(status: InsightDataChecksStatus, findingCount: number): strreturn 'No data to check'case 'inconclusive':return 'Data not checked'+ case 'on_request':+ return 'Data not checked yet'+ case 'partial':+ return 'Data partly checked'case 'warning':return pluralize(findingCount, 'data warning')case 'problem':diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.tsindex c55928284b1a..d806832dcdfe 100644--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts@@ -56,6 +56,7 @@ const SKIP_REASONS: Record<InsightDataCheckSkipReasonEnumApi, string> = {filtered_by_traffic_type: 'this insight already filters on traffic type',hidden_property: 'a property it reads is hidden from you',invalid_filter: "the project's internal and test users filter cannot run",+ on_request: 'it would read a lot of data, so it runs only in a full check',}function eventCount(check: InsightDataCheckApi): string {diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.tsindex e31db25ecfa3..b6e290073a48 100644--- a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts@@ -31,6 +31,7 @@ describe('insightDataChecksLogic', () => {const dataChecks = jest.fn()const cancelledQueryIds: string[] = []let sentSources: Record<string, unknown>[]+ let sentDepths: (string | undefined)[]let savedFilters: unknown[]let capture: jest.SpyInstance@@ -42,8 +43,12 @@ describe('insightDataChecksLogic', () => {const answer =(response: [number, Record<string, unknown>], held?: Promise<void>, onSent?: () => void) =>async ({ request }: { request: Request }): Promise<[number, Record<string, unknown>]> => {- const body = (await request.json()) as { query: { source: Record<string, unknown> } }+ const body = (await request.json()) as {+ query: { source: Record<string, unknown> }+ depth?: string+ }sentSources.push(body.query.source)+ sentDepths.push(body.depth)onSent?.()await heldreturn response@@ -59,6 +64,7 @@ describe('insightDataChecksLogic', () => {dataChecks.mockReset()capture = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined)sentSources = []+ sentDepths = []savedFilters = []cancelledQueryIds.length = 0useMocks({@@ -224,6 +230,37 @@ describe('insightDataChecksLogic', () => {expect(dataChecks).toHaveBeenCalledTimes(requests)})+ it('runs a full check only after a person asks, and only for that query', async () => {+ dataChecks.mockImplementationOnce(answer([200, checksOf('on_request')]))+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))+ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()+ expect(logic.values.canRunFullCheck).toBe(true)++ await expectLogic(logic, () => {+ logic.actions.runFullCheck(logic.values.checkedQuery!)+ }).toFinishAllListeners()+ expect(logic.values).toMatchObject({ status: 'passed', canRunFullCheck: false })++ vizData.actions.updateQuerySource({ dateRange: { date_from: '-30d' } })+ await expectLogic(logic, () => logic.actions.checkData()).toFinishAllListeners()++ expect(sentDepths).toEqual(['quick', 'full', 'quick'])+ })++ it.each([+ ['a check stopped at a limit', 'partial', 'read_limit', true],+ ['every check stopped at a limit', 'inconclusive', 'time_limit', true],+ ['another check found a problem', 'problem', 'memory_limit', true],+ ['the only skipped check could not run for another reason', 'partial', 'hidden_property', false],+ ])('offers a full check of a partly checked chart when %s', async (_name, status, skipReason, offered) => {+ const skipped = { key: 'internal_users', level: 'skipped', skip_reason: skipReason }+ dataChecks.mockImplementation(answer([200, { ...checksOf(status), checks: [skipped] }]))++ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()++ expect(logic.values.canRunFullCheck).toBe(offered)+ })+it.each([['hides the mark from a person who may not run the checks', 403, 'unavailable'],['reports a check the server could not finish', 500, 'failed'],diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.tsindex 0f2f2f86c5fa..450d5b583d8d 100644--- a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts@@ -44,6 +44,7 @@ import { insightsDataChecksCreate } from 'products/product_analytics/frontend/geimport type {InsightDataCheckApi,InsightDataCheckKeyEnumApi,+ InsightDataCheckSkipReasonEnumApi,InsightDataChecksApi,InsightDataChecksRequestApi,InsightDataChecksStatusEnumApi,@@ -110,6 +111,7 @@ export interface insightDataChecksLogicValues {sidePanelOpen: boolean // sidePanelStateLogiccurrentTeam: TeamPublicType | TeamType | null // teamLogiccurrentTeamLoading: boolean // teamLogic+ canRunFullCheck: booleancheckAgainRequested: booleancheckedQuery: DataChecksQuery | nullchecks: InsightDataCheckApi[]@@ -118,6 +120,7 @@ export interface insightDataChecksLogicValues {filterRemovesBots: booleanfiltersTestAccounts: booleanfindings: InsightDataCheckApi[]+ fullCheckQuery: DataChecksQuery | nulllabelHeldOpen: booleanlabelOpen: booleanlastResult: InsightDataChecksApi | null@@ -258,6 +261,9 @@ export interface insightDataChecksLogicActions {reportDataChecksViewed: () => {value: true}+ runFullCheck: (query: DataChecksQuery) => {+ query: DataChecksQuery+ }setFiltersTestAccounts: (filtersTestAccounts: boolean,check: InsightDataCheckKeyEnumApi@@ -294,6 +300,12 @@ export interface insightDataChecksLogicMeta {queryFiltersTestAccounts: (checkedQuery: DataChecksQuery | null) => booleanchecks: (result: InsightDataChecksApi | null) => InsightDataCheckApi[]findings: (checks: InsightDataCheckApi[]) => InsightDataCheckApi[]+ canRunFullCheck: (+ status: InsightDataChecksStatus,+ checks: InsightDataCheckApi[],+ checkedQuery: DataChecksQuery | null,+ fullCheckQuery: DataChecksQuery | null+ ) => booleanlabelOpen: (status: InsightDataChecksStatus, labelHeldOpen: boolean) => boolean}}@@ -306,7 +318,7 @@ export type insightDataChecksLogicType = MakeLogicType<>const FORCED_REFRESHES: (RefreshType | undefined)[] = ['force_async', 'force_blocking']-// The checks run up to four queries of 20 seconds each, after a wait for server slots.+// A full check stops its queries after 60 seconds on the server, which leaves the rest for a wait for server slots.const DATA_CHECKS_TIMEOUT_MS = 120_000const LABEL_OPEN_MS = 2500const YIELDED_TO_CHART = 'yielded to chart'@@ -319,6 +331,13 @@ function readsEvents(series: Node): boolean {return isEventsOrActionsNode(series) || (isGroupNode(series) && series.nodes.every(isEventsOrActionsNode))}+// A full check lifts these limits. It still does not count a day with too many events, so that reason is not here.+const LIMIT_SKIP_REASONS: (InsightDataCheckSkipReasonEnumApi | null)[] = ['read_limit', 'time_limit', 'memory_limit']++function stoppedAtALimit(skipReason: InsightDataCheckSkipReasonEnumApi | null): boolean {+ return LIMIT_SKIP_REASONS.includes(skipReason)+}+export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> = kea<insightDataChecksLogicType>([props({} as InsightLogicProps),key(keyForInsightLogicProps('new')),@@ -365,8 +384,9 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =}),reportCheckExpanded: (check: InsightDataCheckApi) => ({ check }),reportDataChecksViewed: true,+ runFullCheck: (query: DataChecksQuery) => ({ query }),}),- loaders(({ cache }) => ({+ loaders(({ cache, values }) => ({dataChecksRun: [null as DataChecksRun | null,{@@ -395,6 +415,7 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> ={query,client_query_id: clientQueryId,+ depth: values.fullCheckQuery === query ? 'full' : 'quick',} as unknown as InsightDataChecksRequestApi,undefined,{ signal: abortController.signal }@@ -426,6 +447,13 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =loadDataChecks: (_, query) => query,},],+ // A person asks for each query, so a changed query gets a quick check again.+ fullCheckQuery: [+ null as DataChecksQuery | null,+ {+ runFullCheck: (_, { query }) => query,+ },+ ],// A person can refresh the chart during a run. That run is for the events as they were before.checkAgainRequested: [false,@@ -526,6 +554,18 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =(s) => [s.checks],(checks: InsightDataCheckApi[]): InsightDataCheckApi[] => checks.filter(isFinding),],+ canRunFullCheck: [+ (s) => [s.status, s.checks, s.checkedQuery, s.fullCheckQuery],+ (+ status: InsightDataChecksStatus,+ checks: InsightDataCheckApi[],+ checkedQuery: DataChecksQuery | null,+ fullCheckQuery: DataChecksQuery | null+ ): boolean =>+ checkedQuery !== null &&+ checkedQuery !== fullCheckQuery &&+ (status === 'on_request' || checks.some((check) => stoppedAtALimit(check.skip_reason))),+ ],labelOpen: [(s) => [s.status, s.labelHeldOpen],// A finding keeps its label, because the icon alone does not say what the person needs to look at.@@ -580,6 +620,13 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =loadDataSuccess: ({ payload }) => {actions.checkData(FORCED_REFRESHES.includes(payload?.refresh))},+ runFullCheck: ({ query }) => {+ // pinned: analytics event name, renaming breaks dashboards+ posthog.capture('insight data checks full check requested', { status: values.status })+ if (!values.dataChecksRunLoading) {+ actions.loadDataChecks(query)+ }+ },setFiltersTestAccounts: ({ filtersTestAccounts, check }) => {actions.updateQuerySource({ filterTestAccounts: filtersTestAccounts })if (filtersTestAccounts) {diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 420df5f729fb..72340f0c20a1 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -62028,6 +62028,7 @@ export namespace Schemas {* * `filtered_by_traffic_type` - The insight already filters on traffic type* * `hidden_property` - A property the check reads is hidden from you* * `invalid_filter` - The project's internal and test users filter cannot run+ * * `on_request` - The check would read a lot of data, so it runs only on request*/export type InsightDataCheckSkipReasonEnum = typeof InsightDataCheckSkipReasonEnum[keyof typeof InsightDataCheckSkipReasonEnum];@@ -62041,6 +62042,7 @@ export namespace Schemas {FilteredByTrafficType: 'filtered_by_traffic_type',HiddenProperty: 'hidden_property',InvalidFilter: 'invalid_filter',+ OnRequest: 'on_request',} as const;/**@@ -62102,7 +62104,8 @@ export namespace Schemas {* * `too_few_sampled_events` - The sample has too few events to give a share* * `filtered_by_traffic_type` - The insight already filters on traffic type* * `hidden_property` - A property the check reads is hidden from you- * * `invalid_filter` - The project's internal and test users filter cannot run */+ * * `invalid_filter` - The project's internal and test users filter cannot run+ * * `on_request` - The check would read a lot of data, so it runs only on request */skip_reason: InsightDataCheckSkipReasonEnum | null;/** `counted` when the check counted every event on the days checked, `sampled` when it read a sample. Null when the check did not run.*@@ -62167,6 +62170,8 @@ export namespace Schemas {* * `passed` - No data issues found* * `warning` - A check found a small issue* * `problem` - A check found a large issue+ * * `partial` - Some checks could not run, and the others found no issue+ * * `on_request` - The checks would read a lot of data, so they run only on request* * `no_events` - No events to check* * `unsupported` - This kind of insight cannot be checked* * `inconclusive` - No check could run@@ -62178,6 +62183,8 @@ export namespace Schemas {Passed: 'passed',Warning: 'warning',Problem: 'problem',+ Partial: 'partial',+ OnRequest: 'on_request',NoEvents: 'no_events',Unsupported: 'unsupported',Inconclusive: 'inconclusive',@@ -62217,11 +62224,13 @@ export namespace Schemas {}export interface InsightDataChecks {- /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run.+ /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run. `partial` when some checks could not run and the others found nothing. `on_request` when the checks would read a lot of data for a quick check, so they did not run: send `depth` `full` to run them.** * `passed` - No data issues found* * `warning` - A check found a small issue* * `problem` - A check found a large issue+ * * `partial` - Some checks could not run, and the others found no issue+ * * `on_request` - The checks would read a lot of data, so they run only on request* * `no_events` - No events to check* * `unsupported` - This kind of insight cannot be checked* * `inconclusive` - No check could run */@@ -62232,6 +62241,18 @@ export namespace Schemas {coverage: InsightDataChecksCoverage | null;}+ /**+ * * `quick` - Read within a small budget, so the checks can run on every load+ * * `full` - Read with the same limits as the insight's own query+ */+ export type InsightDataChecksDepthEnum = typeof InsightDataChecksDepthEnum[keyof typeof InsightDataChecksDepthEnum];+++ export const InsightDataChecksDepthEnum = {+ Quick: 'quick',+ Full: 'full',+ } as const;+export interface InsightDataChecksRequest {/** The insight query to check, in the shape an insight stores it: an `InsightVizNode` with its `source`. Pass the query of a saved insight, or one that is not saved yet. */query: _InsightQuerySchema;@@ -62240,6 +62261,11 @@ export namespace Schemas {* @maxLength 128*/client_query_id?: string;+ /** `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.+ *+ * * `quick` - Read within a small budget, so the checks can run on every load+ * * `full` - Read with the same limits as the insight's own query */+ depth?: InsightDataChecksDepthEnum;}export interface InsightPlatformAlertCondition {@@ -131215,9 +131241,25 @@ export namespace Schemas {} as const;export type InsightsDataChecksRetrieveParams = {+ /**+ * `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.+ *+ * * `quick` - Read within a small budget, so the checks can run on every load+ * * `full` - Read with the same limits as the insight's own query+ * @minLength 1+ */+ depth?: InsightsDataChecksRetrieveDepth;format?: InsightsDataChecksRetrieveFormat;};+ export type InsightsDataChecksRetrieveDepth = typeof InsightsDataChecksRetrieveDepth[keyof typeof InsightsDataChecksRetrieveDepth];+++ export const InsightsDataChecksRetrieveDepth = {+ Quick: 'quick',+ Full: 'full',+ } as const;+export type InsightsDataChecksRetrieveFormat = typeof InsightsDataChecksRetrieveFormat[keyof typeof InsightsDataChecksRetrieveFormat];diff --git a/services/mcp/src/tools/insights/trustCheck.ts b/services/mcp/src/tools/insights/trustCheck.tsindex 9d477e4ac81b..ae603ee8f0ff 100644--- a/services/mcp/src/tools/insights/trustCheck.ts+++ b/services/mcp/src/tools/insights/trustCheck.ts@@ -35,7 +35,11 @@ type Result = WithAgentNote<Schemas.InsightDataChecks>const AGENT_NOTE ='When a check has the level `warning` or `problem`, tell the user what it found in one sentence, with its share ' +"of the events, and offer the fix. Do not change the insight, the query or the project's filter settings before " +- 'the user agrees. When every check passed, say so in a few words. Do not list the checks that passed.'+ 'the user agrees. When every check passed, say so in a few words. Do not list the checks that passed. When the ' ++ '`status` is `partial`, say which checks did not run and give each `skip_reason` in plain words.'++// An agent's call is already a request to check, so it runs the full check, whatever it reads.+const FULL_DEPTH = { depth: 'full' }const insightTrustCheck = (): ToolBase<typeof schema, Result> => ({name: 'insight-trust-check',@@ -48,11 +52,12 @@ const insightTrustCheck = (): ToolBase<typeof schema, Result> => ({? await context.api.request<Schemas.InsightDataChecks>({method: 'GET',path: `/api/projects/${projectId}/insights/${encodeURIComponent(String(id))}/data_checks/`,+ query: FULL_DEPTH,}): await context.api.request<Schemas.InsightDataChecks>({method: 'POST',path: `/api/projects/${projectId}/insights/data_checks/`,- body: { query },+ body: { query, ...FULL_DEPTH },})return withAgentNote(result, AGENT_NOTE)},