Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(insights): run full data checks only on request (#121973)

Sam Penningtonb11ca07eb1be
diff --git a/products/product_analytics/backend/facade/data_checks.py b/products/product_analytics/backend/facade/data_checks.py
index 7f1985a3490f..88144bae76e8 100644
--- a/products/product_analytics/backend/facade/data_checks.py
+++ b/products/product_analytics/backend/facade/data_checks.py
@@ -13,15 +13,22 @@ from posthog.models.user import User
from products.product_analytics.backend import insight_data_checks
from products.product_analytics.backend.facade.contracts import InsightDataChecks, TeamEmailAccess
+from products.product_analytics.backend.facade.enums import InsightDataChecksDepth
def run_insight_data_checks(
- query: Mapping[str, Any], *, team: Team, user: User, email_access: TeamEmailAccess
+ query: Mapping[str, Any],
+ *,
+ team: Team,
+ user: User,
+ email_access: TeamEmailAccess,
+ depth: InsightDataChecksDepth = InsightDataChecksDepth.QUICK,
) -> InsightDataChecks:
"""Check the events an insight query reads for internal users, local traffic, bots and events sent twice.
The checks read the events as `user`. They do not run when a property they read is hidden from that person.
The internal users check matches only the member emails and verified domains that `email_access` allows, and
names the emails it finds only when `email_access` allows person emails.
+ A quick check does not run when it would read a lot of data. A full check runs with the insight query's own limits.
"""
- return insight_data_checks.run_insight_data_checks(query, team, user, email_access=email_access)
+ return insight_data_checks.run_insight_data_checks(query, team, user, email_access=email_access, depth=depth)
diff --git a/products/product_analytics/backend/facade/enums.py b/products/product_analytics/backend/facade/enums.py
index b5f195b37d35..5265438ff2de 100644
--- a/products/product_analytics/backend/facade/enums.py
+++ b/products/product_analytics/backend/facade/enums.py
@@ -45,6 +45,7 @@ class InsightDataCheckSkipReason(LabeledStrEnum):
FILTERED_BY_TRAFFIC_TYPE = "filtered_by_traffic_type", "The insight already filters on traffic type"
HIDDEN_PROPERTY = "hidden_property", "A property the check reads is hidden from you"
INVALID_FILTER = "invalid_filter", "The project's internal and test users filter cannot run"
+ ON_REQUEST = "on_request", "The check would read a lot of data, so it runs only on request"
class InsightDataCheckAction(LabeledStrEnum):
@@ -52,10 +53,17 @@ class InsightDataCheckAction(LabeledStrEnum):
OPEN_FILTER_SETTINGS = "open_filter_settings", "Change the project's internal and test users filter"
+class InsightDataChecksDepth(LabeledStrEnum):
+ QUICK = "quick", "Read within a small budget, so the checks can run on every load"
+ FULL = "full", "Read with the same limits as the insight's own query"
+
+
class InsightDataChecksStatus(LabeledStrEnum):
PASSED = "passed", "No data issues found"
WARNING = "warning", "A check found a small issue"
PROBLEM = "problem", "A check found a large issue"
+ PARTIAL = "partial", "Some checks could not run, and the others found no issue"
+ ON_REQUEST = "on_request", "The checks would read a lot of data, so they run only on request"
NO_EVENTS = "no_events", "No events to check"
UNSUPPORTED = "unsupported", "This kind of insight cannot be checked"
INCONCLUSIVE = "inconclusive", "No check could run"
diff --git a/products/product_analytics/backend/insight_data_check_results.py b/products/product_analytics/backend/insight_data_check_results.py
index 6666e1d023c0..3fcd3cbdfc8f 100644
--- a/products/product_analytics/backend/insight_data_check_results.py
+++ b/products/product_analytics/backend/insight_data_check_results.py
@@ -23,6 +23,10 @@ PROBLEM_SHARE = 0.2
# An event with no user agent has the type "Automation", and server-side SDKs send those, so it is not a crawler.
CRAWLER_TRAFFIC_TYPES = ("Bot", "AI Agent")
+_SKIP_REASONS_WITH_NOTHING_TO_CHECK = frozenset(
+ {InsightDataCheckSkipReason.FILTERED_BY_TRAFFIC_TYPE, InsightDataCheckSkipReason.TOO_FEW_SAMPLED_EVENTS}
+)
+
LOCAL_HOST_PATTERN = r"(?i)^(localhost|[a-z0-9.-]+\.localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\])(:[0-9]+)?$"
@@ -259,4 +263,9 @@ def status_of(checks: list[InsightDataCheck]) -> InsightDataChecksStatus:
return InsightDataChecksStatus.PROBLEM
if InsightDataCheckLevel.WARNING in levels:
return InsightDataChecksStatus.WARNING
+ if any(
+ check.level == InsightDataCheckLevel.SKIPPED and check.skip_reason not in _SKIP_REASONS_WITH_NOTHING_TO_CHECK
+ for check in checks
+ ):
+ return InsightDataChecksStatus.PARTIAL
return InsightDataChecksStatus.PASSED
diff --git a/products/product_analytics/backend/insight_data_checks.py b/products/product_analytics/backend/insight_data_checks.py
index 3435b1c99588..d306b014eeba 100644
--- a/products/product_analytics/backend/insight_data_checks.py
+++ b/products/product_analytics/backend/insight_data_checks.py
@@ -1,20 +1,24 @@
"""Checks the events an insight reads for internal users, local development, bots and events sent twice."""
+import math
from collections.abc import Mapping
from datetime import date, datetime, time, timedelta
+from time import monotonic
from typing import Any, cast
from zoneinfo import ZoneInfo
-from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
+from posthog.schema import HogQLQueryModifiers, InlineCohortCalculation, PersonsOnEventsMode
from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
+from posthog.hogql.database.models import DatabaseField
+from posthog.hogql.database.schema.events import EventsTable
from posthog.hogql.errors import ExposedHogQLError, ResolutionError
from posthog.hogql.parser import parse_expr, parse_select
from posthog.hogql.property import property_to_expr
-from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.query import HogQLQueryExecutor, execute_hogql_query
from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
@@ -43,6 +47,7 @@ from products.product_analytics.backend.facade.contracts import (
)
from products.product_analytics.backend.facade.enums import (
InsightDataCheckKey,
+ InsightDataChecksDepth,
InsightDataCheckSkipReason,
InsightDataChecksStatus,
)
@@ -80,6 +85,14 @@ MAX_EXECUTION_SECONDS = 5
MAX_BYTES_READ = 10_000_000_000
# The sample reads wide columns and sorts them for the repeat window, so it gets a tighter limit.
MAX_SAMPLE_BYTES_READ = 1_000_000_000
+MAX_ESTIMATED_BYTES = 10_000_000_000
+# Larger than the JSON of almost any event. Below it, the checks skip the size read, which can read hundreds of MB.
+# An event above it only makes a check stop at MAX_BYTES_READ, which offers the person the same button.
+MAX_JSON_BYTES_PER_EVENT = 10_000
+# A full check has the same time limit as the insight's own query, and no read limit. Its queries share the limit,
+# so the whole run answers before the page and the MCP client stop waiting at 120 seconds.
+FULL_MAX_EXECUTION_SECONDS = 60
+JSON_SIZE_SAMPLE = 1_000
# A person cannot repeat an action this fast, but an app that sends an event twice does.
REPEAT_WINDOW_MILLISECONDS = 100
@@ -198,6 +211,38 @@ _SAMPLE_COUNTS_QUERY = """
)
"""
+_JSON_SIZE_QUERY = """
+ SELECT avg(json_bytes)
+ FROM {sampled_events}
+"""
+
+_JSON_COLUMN_FIELDS = {"properties": ("properties",), "person_properties": ("person", "properties")}
+# A property group map holds part of a JSON column, so the size of that JSON column bounds what reading the map costs.
+_PROPERTY_MAP_PREFIXES = {"properties_group_": "properties", "person_properties_map_": "person_properties"}
+
+
+@frozen
+class _Limits:
+ max_execution_seconds: int
+ max_bytes_read: int | None
+ max_sample_bytes_read: int | None
+ deadline: float | None = None
+
+
+def _limits(depth: InsightDataChecksDepth) -> _Limits:
+ if depth == InsightDataChecksDepth.FULL:
+ return _Limits(
+ max_execution_seconds=FULL_MAX_EXECUTION_SECONDS,
+ max_bytes_read=None,
+ max_sample_bytes_read=None,
+ deadline=monotonic() + FULL_MAX_EXECUTION_SECONDS,
+ )
+ return _Limits(
+ max_execution_seconds=MAX_EXECUTION_SECONDS,
+ max_bytes_read=MAX_BYTES_READ,
+ max_sample_bytes_read=MAX_SAMPLE_BYTES_READ,
+ )
+
@frozen
class _EventReader:
@@ -206,10 +251,19 @@ class _EventReader:
modifiers: HogQLQueryModifiers
# The database the chart's events were read with, so the checks do not build another.
database: Database
+ limits: _Limits
+
+ def context(self) -> HogQLContext:
+ return HogQLContext(team_id=self.team.pk, user=self.user, database=self.database)
def rows_within_limits(
- self, query: ast.SelectQuery, *, query_type: str, max_bytes_read: int
+ self, query: ast.SelectQuery, *, query_type: str, max_bytes_read: int | None, max_threads: int | None = None
) -> list[list[Any]] | Skipped:
+ max_execution_seconds = self.limits.max_execution_seconds
+ if self.limits.deadline is not None:
+ max_execution_seconds = min(max_execution_seconds, math.floor(self.limits.deadline - monotonic()))
+ if max_execution_seconds < 1:
+ return Skipped(reason=InsightDataCheckSkipReason.TIME_LIMIT)
try:
response = execute_hogql_query(
query=query,
@@ -217,8 +271,8 @@ class _EventReader:
user=self.user,
query_type=query_type,
modifiers=self.modifiers,
- context=HogQLContext(team_id=self.team.pk, user=self.user, database=self.database),
- settings=_query_settings(max_bytes_read),
+ context=self.context(),
+ settings=_query_settings(max_bytes_read, max_execution_seconds, max_threads),
)
except CHQueryErrorTooManyBytes:
return Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT)
@@ -229,13 +283,16 @@ class _EventReader:
return [list(row) for row in response.results or []]
-def _query_settings(max_bytes_read: int) -> HogQLGlobalSettings:
+def _query_settings(
+ max_bytes_read: int | None, max_execution_seconds: int, max_threads: int | None
+) -> HogQLGlobalSettings:
# Without "throw", a query that reaches a limit can return the events it counted so far as the answer.
return HogQLGlobalSettings(
- max_execution_time=MAX_EXECUTION_SECONDS,
+ max_execution_time=max_execution_seconds,
timeout_overflow_mode="throw",
max_bytes_to_read=max_bytes_read,
- read_overflow_mode="throw",
+ read_overflow_mode="throw" if max_bytes_read is not None else None,
+ max_threads=max_threads,
)
@@ -262,7 +319,12 @@ def _check_modifiers(chart_modifiers: HogQLQueryModifiers, *, keep_project_bot_r
# With `debug` on, a failed query returns an error and no rows, which would read as no events.
# Without the pushdown, a persons join reads the latest version of every person in the project, however few
# events the check reads.
- update: dict[str, Any] = {"debug": False, "personIdPushdown": True}
+ # An inlined cohort scans every person in the project, so a cohort in a filter reads its stored members instead.
+ update: dict[str, Any] = {
+ "debug": False,
+ "personIdPushdown": True,
+ "inlineCohortCalculation": InlineCohortCalculation.OFF,
+ }
if not keep_project_bot_rules:
# The bot check uses PostHog's definition of a bot, so a project's own bot rules do not apply.
update["customBotDefinitions"] = []
@@ -326,7 +388,9 @@ def _count_events_per_day(
where=ast.And(exprs=[events.sort_key_where, _between_days(events, days[0], days[-1], timezone)]),
sample=_sample_of(1, SIZING_SAMPLE_ONE_IN),
)
- rows = reader.rows_within_limits(query, query_type="insight_data_checks_sizing", max_bytes_read=MAX_BYTES_READ)
+ rows = reader.rows_within_limits(
+ query, query_type="insight_data_checks_sizing", max_bytes_read=reader.limits.max_bytes_read
+ )
if isinstance(rows, Skipped):
return rows
@@ -508,9 +572,9 @@ def _internal_users_match(
)
-def _count_every_event(
- events: ChartEvents, reader: _EventReader, where: ast.Expr, internal_users: _InternalUsersMatch | Skipped
-) -> CountedEvents | Skipped:
+def _counted_events_query(
+ events: ChartEvents, where: ast.Expr, internal_users: _InternalUsersMatch | Skipped
+) -> ast.SelectQuery:
match = _NO_INTERNAL_USERS if isinstance(internal_users, Skipped) else internal_users
query = events.select(
_COUNTED_EVENTS_QUERY,
@@ -522,7 +586,16 @@ def _count_every_event(
)
if match.person_fields is not None:
cast(ast.JoinExpr, query.select_from).next_join = _current_person_join(events, where, match.person_fields)
- rows = reader.rows_within_limits(query, query_type="insight_data_checks_counted", max_bytes_read=MAX_BYTES_READ)
+ return query
+
+
+def _count_every_event(
+ events: ChartEvents, reader: _EventReader, where: ast.Expr, internal_users: _InternalUsersMatch | Skipped
+) -> CountedEvents | Skipped:
+ query = _counted_events_query(events, where, internal_users)
+ rows = reader.rows_within_limits(
+ query, query_type="insight_data_checks_counted", max_bytes_read=reader.limits.max_bytes_read
+ )
if isinstance(rows, Skipped):
return rows
@@ -560,7 +633,9 @@ def _count_local_events_the_filter_keeps(
if person_fields is not None:
local_events = ast.And(exprs=[clone_expr(where), clone_expr(is_local_host)])
cast(ast.JoinExpr, query.select_from).next_join = _current_person_join(events, local_events, person_fields)
- rows = reader.rows_within_limits(query, query_type="insight_data_checks_local", max_bytes_read=MAX_BYTES_READ)
+ rows = reader.rows_within_limits(
+ query, query_type="insight_data_checks_local", max_bytes_read=reader.limits.max_bytes_read
+ )
if isinstance(rows, Skipped):
return rows
@@ -624,7 +699,7 @@ def _sample_events(
rows = reader.rows_within_limits(
cast(ast.SelectQuery, query),
query_type="insight_data_checks_sample",
- max_bytes_read=MAX_SAMPLE_BYTES_READ,
+ max_bytes_read=reader.limits.max_sample_bytes_read,
)
if isinstance(rows, Skipped):
return rows
@@ -644,6 +719,12 @@ def _sample_events(
)
+def _checked_where(events: ChartEvents, window: CheckWindow, timezone: ZoneInfo) -> ast.Expr:
+ # The bounds apply to a whole range too, because a lifecycle chart also reads the interval before its range.
+ first, last = window.days_to_check[0], window.days_to_check[-1]
+ return ast.And(exprs=[events.where, _between_days(events, first, last, timezone)])
+
+
def _read_checked_days(
events: ChartEvents,
reader: _EventReader,
@@ -654,9 +735,7 @@ def _read_checked_days(
filters_traffic_type: bool,
email_access: TeamEmailAccess,
) -> CheckedEvents:
- # The bounds apply to a whole range too, because a lifecycle chart also reads the interval before its range.
- first, last = window.days_to_check[0], window.days_to_check[-1]
- where = ast.And(exprs=[events.where, _between_days(events, first, last, timezone)])
+ where = _checked_where(events, window, timezone)
counted: CountedEvents | Skipped = Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS)
if window.events_on_days_to_check <= MAX_COUNTED_EVENTS:
@@ -668,13 +747,130 @@ def _read_checked_days(
return CheckedEvents(counted=counted, local=local, sample=sample, insight_filters_traffic_type=filters_traffic_type)
+def _reads_the_events_table(field_type: ast.FieldType) -> bool:
+ table_type: ast.Type | None = field_type.table_type
+ while isinstance(table_type, ast.TableAliasType | ast.ColumnAliasedTableType | ast.VirtualTableType):
+ table_type = table_type.table_type
+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)
+
+
+class _EventJsonReads(TraversingVisitor):
+ """The JSON columns of the events table that a printed query reads a property from."""
+
+ def __init__(self, context: HogQLContext) -> None:
+ super().__init__()
+ self.context = context
+ self.columns: set[str] = set()
+
+ # A property read and a key-existence test both reach the JSON column through a field, so the field is enough.
+ def visit_field(self, node: ast.Field) -> None:
+ name = str(node.chain[-1])
+ self.columns.update(column for prefix, column in _PROPERTY_MAP_PREFIXES.items() if name.startswith(prefix))
+ field_type = node.type
+ if (
+ isinstance(field_type, ast.FieldType)
+ and field_type.name in _JSON_COLUMN_FIELDS
+ and _reads_the_events_table(field_type)
+ ):
+ column = field_type.resolve_database_field(self.context)
+ if isinstance(column, DatabaseField) and column.name in _JSON_COLUMN_FIELDS:
+ self.columns.add(column.name)
+ super().visit_field(node)
+
+ # A native JSON column stores each path as a subcolumn, so this read is narrow, however wide the document.
+ def visit_json_subcolumn_access(self, node: ast.JsonSubcolumnAccess) -> None:
+ return
+
+
+def _json_columns_read(query: ast.SelectQuery, reader: _EventReader) -> frozenset[str]:
+ """The JSON columns `query` reads a property from, decided as the query is printed to run."""
+ executor = HogQLQueryExecutor(
+ query=clone_expr(query),
+ team=reader.team,
+ user=reader.user,
+ modifiers=reader.modifiers,
+ context=reader.context(),
+ )
+ executor.generate_clickhouse_sql()
+ if executor.clickhouse_prepared_ast is None or executor.clickhouse_context is None:
+ return frozenset()
+ reads = _EventJsonReads(executor.clickhouse_context)
+ reads.visit(executor.clickhouse_prepared_ast)
+ return frozenset(reads.columns)
+
+
+def _json_bytes_per_event(
+ events: ChartEvents, reader: _EventReader, where: ast.Expr, columns: frozenset[str]
+) -> float | Skipped:
+ """The average size of the JSON columns `columns` across a few of the events `where` keeps."""
+ json_bytes = parse_expr(
+ " + ".join(f"length({{{column}}})" for column in sorted(columns)),
+ placeholders={column: ast.Field(chain=list(_JSON_COLUMN_FIELDS[column])) for column in columns},
+ )
+ sampled_events = events.select("SELECT {json_bytes} AS json_bytes FROM events", where=where, json_bytes=json_bytes)
+ sampled_events.limit = ast.Constant(value=JSON_SIZE_SAMPLE)
+ query = parse_select(_JSON_SIZE_QUERY, placeholders={"sampled_events": sampled_events})
+ rows = reader.rows_within_limits(
+ cast(ast.SelectQuery, query),
+ query_type="insight_data_checks_json_size",
+ max_bytes_read=reader.limits.max_bytes_read,
+ # Each thread reads at least one block of the JSON column before the limit stops it, so one thread reads least.
+ max_threads=1,
+ )
+ if isinstance(rows, Skipped):
+ return rows
+ # The average of no events is NaN, and no events cost nothing to read.
+ average = float(rows[0][0] or 0) if rows else 0.0
+ return 0.0 if math.isnan(average) else average
+
+
+def _would_read_too_much(
+ events: ChartEvents,
+ reader: _EventReader,
+ window: CheckWindow,
+ timezone: ZoneInfo,
+ *,
+ email_access: TeamEmailAccess,
+ hidden_properties: RestrictedPropertyNames,
+) -> bool:
+ """Whether the count of every event, which is the most any check reads, would read more than MAX_ESTIMATED_BYTES.
+
+ Only a property without a column of its own makes a check heavy, because it is read from a JSON column of every
+ event the check reads.
+ """
+ if window.events_on_days_to_check * MAX_JSON_BYTES_PER_EVENT <= MAX_ESTIMATED_BYTES:
+ return False
+
+ where = _checked_where(events, window, timezone)
+ try:
+ internal_users = _internal_users_match(
+ events, reader, email_access=email_access, hidden_properties=hidden_properties
+ )
+ columns = _json_columns_read(_counted_events_query(events, where, internal_users), reader)
+ except (ExposedHogQLError, ResolutionError):
+ # The count leaves out a project filter that cannot run, so the estimate leaves it out too.
+ no_filter = Skipped(reason=InsightDataCheckSkipReason.INVALID_FILTER)
+ columns = _json_columns_read(_counted_events_query(events, where, no_filter), reader)
+ if not columns:
+ return False
+
+ first, last = window.days_to_check[0], window.days_to_check[-1]
+ # The count reads each event of the chart's event names and days before its filters drop any.
+ sort_key_where = ast.And(exprs=[events.sort_key_where, _between_days(events, first, last, timezone)])
+ bytes_per_event = _json_bytes_per_event(events, reader, sort_key_where, columns)
+ # A size read that stops at a limit means the full count would stop there too.
+ if isinstance(bytes_per_event, Skipped):
+ return True
+ return window.events_on_days_to_check * bytes_per_event > MAX_ESTIMATED_BYTES
+
+
def _sample_summary(sample: SampledEvents | Skipped) -> InsightDataChecksSample | None:
if isinstance(sample, Skipped):
return None
return InsightDataChecksSample(rate=sample.rate, events=sample.events, distinct_ids=sample.distinct_ids)
-def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents) -> InsightDataChecksCoverage:
+def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents | None) -> InsightDataChecksCoverage:
return InsightDataChecksCoverage(
date_from=window.days_in_range[0],
date_to=window.days_in_range[-1],
@@ -683,8 +879,8 @@ def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents)
checked_date_to=window.days_to_check[-1],
days_checked=len(window.days_to_check),
filters_test_accounts=events.filters_test_accounts,
- events_checked=events_on_checked_days(checked),
- sample=_sample_summary(checked.sample),
+ events_checked=events_on_checked_days(checked) if checked else None,
+ sample=_sample_summary(checked.sample) if checked else None,
)
@@ -698,7 +894,12 @@ def _nothing_checked(
def run_insight_data_checks(
- query: Mapping[str, Any], team: Team, user: User, *, email_access: TeamEmailAccess
+ query: Mapping[str, Any],
+ team: Team,
+ user: User,
+ *,
+ email_access: TeamEmailAccess,
+ depth: InsightDataChecksDepth = InsightDataChecksDepth.QUICK,
) -> InsightDataChecks:
with tags_context(product=Product.PRODUCT_ANALYTICS, feature=Feature.INSIGHT, name="insight_data_checks"):
events = chart_events(query, team, user)
@@ -715,11 +916,20 @@ def run_insight_data_checks(
user=user,
modifiers=_check_modifiers(events.modifiers, keep_project_bot_rules=filters_traffic_type),
database=events.database,
+ limits=_limits(depth),
)
timezone = ZoneInfo(team.timezone)
window = _choose_window(events, reader, timezone)
if isinstance(window, Skipped):
return _nothing_checked(InsightDataChecksStatus.INCONCLUSIVE, reason=window.reason)
+ if depth == InsightDataChecksDepth.QUICK and _would_read_too_much(
+ events, reader, window, timezone, email_access=email_access, hidden_properties=hidden_properties
+ ):
+ return _nothing_checked(
+ InsightDataChecksStatus.ON_REQUEST,
+ _coverage(events, window, None),
+ InsightDataCheckSkipReason.ON_REQUEST,
+ )
checked = _read_checked_days(
events,
reader,
diff --git a/products/product_analytics/backend/presentation/insight.py b/products/product_analytics/backend/presentation/insight.py
index 1e44cd43e14c..59d1950ef88b 100644
--- a/products/product_analytics/backend/presentation/insight.py
+++ b/products/product_analytics/backend/presentation/insight.py
@@ -187,6 +187,7 @@ from products.product_analytics.backend.facade.enums import (
InsightDataCheckKey,
InsightDataCheckLevel,
InsightDataCheckMethod,
+ InsightDataChecksDepth,
InsightDataCheckSkipReason,
InsightDataChecksStatus,
)
@@ -1698,6 +1699,13 @@ class InsightBulkSetTestAccountFilterResponseSerializer(serializers.Serializer):
)
+DEPTH_HELP_TEXT = (
+ "`quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would "
+ "read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the "
+ "same limits as the insight's own query."
+)
+
+
class InsightDataChecksRequestSerializer(serializers.Serializer):
query = QueryFieldSerializer(
help_text=(
@@ -1713,6 +1721,21 @@ class InsightDataChecksRequestSerializer(serializers.Serializer):
"`query/{client_query_id}/`."
),
)
+ depth = serializers.ChoiceField(
+ choices=InsightDataChecksDepth.choices,
+ required=False,
+ default=InsightDataChecksDepth.QUICK,
+ help_text=DEPTH_HELP_TEXT,
+ )
+
+
+class InsightDataChecksQueryParamsSerializer(serializers.Serializer):
+ depth = serializers.ChoiceField(
+ choices=InsightDataChecksDepth.choices,
+ required=False,
+ default=InsightDataChecksDepth.QUICK,
+ help_text=DEPTH_HELP_TEXT,
+ )
class InsightDataCheckFilterSerializer(serializers.Serializer):
@@ -1876,7 +1899,9 @@ class InsightDataChecksSerializer(serializers.Serializer):
help_text=(
"The worst result across the checks. `no_events` when no event matches the insight's events, filters "
"and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, "
- "and when a property the checks read is hidden from the caller. `inconclusive` when no check could run."
+ "and when a property the checks read is hidden from the caller. `inconclusive` when no check could run. "
+ "`partial` when some checks could not run and the others found nothing. `on_request` when the checks "
+ "would read a lot of data for a quick check, so they did not run: send `depth` `full` to run them."
),
)
checks = InsightDataCheckSerializer(many=True, help_text="Every check, in a fixed order, whatever its result.")
@@ -2654,9 +2679,10 @@ When set, the specified dashboard's filters and date range override will be appl
raise PermissionDenied("You need query access to check a query that is not saved as an insight.")
if client_query_id := request.validated_data.get("client_query_id"):
tag_queries(client_query_id=client_query_id)
- return self._data_checks_response(request.validated_data["query"])
+ return self._data_checks_response(request.validated_data["query"], depth=request.validated_data["depth"])
@validated_request(
+ query_serializer=InsightDataChecksQueryParamsSerializer,
parameters=[INSIGHT_ID_PATH_PARAMETER],
responses={
200: OpenApiResponse(response=InsightDataChecksSerializer, description="The result of every check.")
@@ -2671,9 +2697,12 @@ When set, the specified dashboard's filters and date range override will be appl
@action(methods=["GET"], detail=True, url_path="data_checks", required_scopes=["insight:read"])
def saved_insight_data_checks(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response:
insight = self.get_object()
- return self._data_checks_response(upgrade(insight.query) if insight.query else {})
+ return self._data_checks_response(
+ upgrade(insight.query) if insight.query else {},
+ depth=request.validated_query_data["depth"],
+ )
- def _data_checks_response(self, query: dict[str, Any]) -> Response:
+ def _data_checks_response(self, query: dict[str, Any], *, depth: InsightDataChecksDepth) -> Response:
# A personal API key reads within the same hourly budget as the query API, and each check query debits it.
if get_query_tag_value("access_method") == AccessMethod.PERSONAL_API_KEY:
tag_queries(api_queries_budgeted=True)
@@ -2685,6 +2714,7 @@ When set, the specified dashboard's filters and date range override will be appl
team=self.team,
user=cast(User, self.request.user),
email_access=self._team_email_access(),
+ depth=InsightDataChecksDepth(depth),
)
except CHQueryErrorQueryWasCancelled:
# The client cancelled the run, so nothing reads a body. 499 is Nginx's "Client Closed Request".
diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.py
index bd496806a12d..e1b84bd77408 100644
--- a/products/product_analytics/backend/tests/test_insight_data_checks.py
+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py
@@ -6,9 +6,17 @@ from types import MappingProxyType
from typing import Any
import time_machine
-from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events
+from posthog.test.base import (
+ APIBaseTest,
+ ClickhouseTestMixin,
+ _create_event,
+ _create_person,
+ cleanup_materialized_columns,
+ flush_persons_and_events,
+)
from unittest.mock import patch
+from django.conf import settings
from django.test import SimpleTestCase
from parameterized import parameterized
@@ -37,6 +45,7 @@ from products.product_analytics.backend.facade.data_checks import run_insight_da
from products.product_analytics.backend.facade.enums import (
InsightDataCheckKey,
InsightDataCheckLevel,
+ InsightDataChecksDepth,
InsightDataCheckSkipReason,
)
from products.product_analytics.backend.facade.models import Insight
@@ -53,6 +62,8 @@ from products.product_analytics.backend.insight_data_check_results import (
)
from products.product_analytics.backend.insight_data_check_team_emails import TeamEmails, team_email_pattern
+from ee.clickhouse.materialized_columns.columns import materialize
+
PASSED = InsightDataCheckLevel.PASSED
WARNING = InsightDataCheckLevel.WARNING
PROBLEM = InsightDataCheckLevel.PROBLEM
@@ -78,6 +89,7 @@ FROM_A_BOT = {"key": "$raw_user_agent", "value": [BOT_USER_AGENT], "operator": "
NOT_A_BOT = {"key": "$virt_traffic_type", "value": ["Bot"], "operator": "is_not", "type": "event"}
IS_PAID = {"key": "plan", "value": ["paid"], "operator": "exact", "type": "event"}
HAS_NO_PLAN_TODAY = {"key": "plan", "value": "is_not_set", "operator": "is_not_set", "type": "person"}
+SENT_WITHOUT_A_PLAN = {"key": "plan", "value": "is_not_set", "operator": "is_not_set", "type": "event"}
ONE_STEP = [{"kind": "EventsNode", "event": EVENT}]
TWO_STEPS = [{"kind": "EventsNode", "event": EVENT}, {"kind": "EventsNode", "event": EVENT}]
SEVEN_DAYS = [date(2026, 3, 1) + timedelta(days=offset) for offset in range(7)]
@@ -215,16 +227,33 @@ class TestEvaluateDataChecks(SimpleTestCase):
internal_users = next(check for check in checks if check.key == INTERNAL)
assert (internal_users.events, internal_users.action) == (removed_by_filter + missed_by_filter, action)
- def test_status_is_inconclusive_when_every_check_is_skipped(self) -> None:
- checks = evaluate_data_checks(
- CheckedEvents(
- counted=Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS),
- local=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),
- sample=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),
- )
- )
-
- assert status_of(checks) == "inconclusive"
+ @parameterized.expand(
+ [
+ (
+ "every_check_skipped",
+ CheckedEvents(
+ counted=Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS),
+ local=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),
+ sample=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),
+ ),
+ "inconclusive",
+ ),
+ (
+ "sample_hit_the_read_limit",
+ replace(checked(1000), sample=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT)),
+ "partial",
+ ),
+ (
+ "bot_check_not_needed_because_the_insight_filters_on_traffic_type",
+ replace(checked(1000), insight_filters_traffic_type=True),
+ "passed",
+ ),
+ ]
+ )
+ def test_status_says_whether_every_check_ran(
+ self, _name: str, checked_events: CheckedEvents, expected: str
+ ) -> None:
+ assert status_of(evaluate_data_checks(checked_events)) == expected
class TestNewestDaysWithin(SimpleTestCase):
@@ -324,12 +353,15 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):
},
}
- def _check(self, query: dict[str, Any]) -> InsightDataChecks:
+ def _check(
+ self, query: dict[str, Any], depth: InsightDataChecksDepth = InsightDataChecksDepth.QUICK
+ ) -> InsightDataChecks:
return run_insight_data_checks(
query,
team=self.team,
user=self.user,
email_access=TeamEmailAccess(member_emails=True, verified_domains=True, person_emails=True),
+ depth=depth,
)
def _internal_users(self, result: InsightDataChecks) -> InsightDataCheck:
@@ -749,19 +781,73 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):
@parameterized.expand(
[
- ("unknown_event", [{"kind": "EventsNode", "event": "never sent"}], 1_000_000_000, "no_events"),
- ("read_limit_reached_while_sizing", ONE_STEP, 1, "inconclusive"),
+ ("unknown_event", [{"kind": "EventsNode", "event": "never sent"}], 1_000_000_000, "quick", "no_events"),
+ ("read_limit_reached_while_sizing", ONE_STEP, 1, "quick", "inconclusive"),
+ ("full_check_past_its_deadline", ONE_STEP, 1_000_000_000, "full", "inconclusive"),
]
)
+ @patch(f"{CHECKS_MODULE}.FULL_MAX_EXECUTION_SECONDS", 0)
def test_reports_why_nothing_was_checked(
- self, _name: str, series: list[dict[str, Any]], max_bytes_read: int, expected: str
+ self, _name: str, series: list[dict[str, Any]], max_bytes_read: int, depth: str, expected: str
) -> None:
with patch(f"{CHECKS_MODULE}.MAX_BYTES_READ", max_bytes_read):
- result = self._check(self._query(series))
+ result = self._check(self._query(series), InsightDataChecksDepth(depth))
assert result.status == expected
assert {check.level for check in result.checks} == {"skipped"}
+ # The native events table reads a property from a subcolumn of its own, so a property filter reads little there.
+ @parameterized.expand(
+ [
+ ("properties_with_columns_of_their_own", [], None, 10_000, ("warning", "warning")),
+ (
+ "a_filter_on_a_property_without_a_column",
+ [SENT_WITHOUT_A_PLAN],
+ None,
+ 10_000,
+ ("on_request", "warning"),
+ ),
+ (
+ "too_few_events_to_read_a_lot_even_from_large_events",
+ [SENT_WITHOUT_A_PLAN],
+ None,
+ 0,
+ ("warning", "warning"),
+ ),
+ ("asked_for_in_the_request_body", [SENT_WITHOUT_A_PLAN], "body", 10_000, ("warning", "warning")),
+ ("asked_for_on_a_saved_insight", [SENT_WITHOUT_A_PLAN], "saved_insight", 10_000, ("warning", "warning")),
+ ]
+ )
+ @patch(f"{CHECKS_MODULE}.MAX_ESTIMATED_BYTES", 1)
+ def test_runs_the_checks_that_would_read_a_lot_only_on_request(
+ self,
+ _name: str,
+ properties: list[dict[str, Any]],
+ asked_for: str | None,
+ max_json_bytes_per_event: int,
+ status_by_events_schema: tuple[str, str],
+ ) -> None:
+ legacy_status, native_status = status_by_events_schema
+ expected_status = native_status if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else legacy_status
+ self.enterContext(patch(f"{CHECKS_MODULE}.MAX_JSON_BYTES_PER_EVENT", max_json_bytes_per_event))
+ self.addCleanup(cleanup_materialized_columns)
+ materialize("events", "email", table_column="person_properties")
+ # A lifecycle chart filters on this property, which production reads from a column of its own.
+ materialize("events", "$process_person_profile")
+ query = self._query(
+ properties=properties, modifiers={"personsOnEventsMode": "person_id_override_properties_on_events"}
+ )
+
+ if asked_for == "saved_insight":
+ insight = Insight.objects.create(team=self.team, query=query)
+ response = self.client.get(f"/api/projects/{self.team.id}/insights/{insight.id}/data_checks/?depth=full")
+ else:
+ body = {"query": query, "depth": "full"} if asked_for == "body" else {"query": query}
+ response = self.client.post(f"/api/projects/{self.team.id}/insights/data_checks/", body, format="json")
+
+ assert response.status_code == status.HTTP_200_OK, response.content
+ assert response.json()["status"] == expected_status
+
def test_a_filter_that_cannot_run_raises_with_the_debug_modifier_on(self) -> None:
query = self._query(properties=[{"type": "hogql", "key": "throwIf(1, 'stop') = 0"}], modifiers={"debug": True})
diff --git a/products/product_analytics/frontend/generated/api.schemas.ts b/products/product_analytics/frontend/generated/api.schemas.ts
index 1038a9caa8ad..a252e090a59b 100644
--- a/products/product_analytics/frontend/generated/api.schemas.ts
+++ b/products/product_analytics/frontend/generated/api.schemas.ts
@@ -9072,6 +9072,8 @@ export interface ActivityLogPaginatedResponseApi {
* * `passed` - No data issues found
* * `warning` - A check found a small issue
* * `problem` - A check found a large issue
+ * * `partial` - Some checks could not run, and the others found no issue
+ * * `on_request` - The checks would read a lot of data, so they run only on request
* * `no_events` - No events to check
* * `unsupported` - This kind of insight cannot be checked
* * `inconclusive` - No check could run
@@ -9083,6 +9085,8 @@ export const InsightDataChecksStatusEnumApi = {
Passed: 'passed',
Warning: 'warning',
Problem: 'problem',
+ Partial: 'partial',
+ OnRequest: 'on_request',
NoEvents: 'no_events',
Unsupported: 'unsupported',
Inconclusive: 'inconclusive',
@@ -9128,6 +9132,7 @@ export const InsightDataCheckLevelEnumApi = {
* * `filtered_by_traffic_type` - The insight already filters on traffic type
* * `hidden_property` - A property the check reads is hidden from you
* * `invalid_filter` - The project's internal and test users filter cannot run
+ * * `on_request` - The check would read a lot of data, so it runs only on request
*/
export type InsightDataCheckSkipReasonEnumApi =
(typeof InsightDataCheckSkipReasonEnumApi)[keyof typeof InsightDataCheckSkipReasonEnumApi]
@@ -9141,6 +9146,7 @@ export const InsightDataCheckSkipReasonEnumApi = {
FilteredByTrafficType: 'filtered_by_traffic_type',
HiddenProperty: 'hidden_property',
InvalidFilter: 'invalid_filter',
+ OnRequest: 'on_request',
} as const
/**
@@ -9202,7 +9208,8 @@ export interface InsightDataCheckApi {
* * `too_few_sampled_events` - The sample has too few events to give a share
* * `filtered_by_traffic_type` - The insight already filters on traffic type
* * `hidden_property` - A property the check reads is hidden from you
- * * `invalid_filter` - The project's internal and test users filter cannot run */
+ * * `invalid_filter` - The project's internal and test users filter cannot run
+ * * `on_request` - The check would read a lot of data, so it runs only on request */
skip_reason: InsightDataCheckSkipReasonEnumApi | null
/** `counted` when the check counted every event on the days checked, `sampled` when it read a sample. Null when the check did not run.
*
@@ -9297,11 +9304,13 @@ export interface InsightDataChecksCoverageApi {
}
export interface InsightDataChecksApi {
- /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run.
+ /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run. `partial` when some checks could not run and the others found nothing. `on_request` when the checks would read a lot of data for a quick check, so they did not run: send `depth` `full` to run them.
*
* * `passed` - No data issues found
* * `warning` - A check found a small issue
* * `problem` - A check found a large issue
+ * * `partial` - Some checks could not run, and the others found no issue
+ * * `on_request` - The checks would read a lot of data, so they run only on request
* * `no_events` - No events to check
* * `unsupported` - This kind of insight cannot be checked
* * `inconclusive` - No check could run */
@@ -9418,6 +9427,18 @@ export interface BulkUpdateTagsResponseApi {
skipped: BulkUpdateTagsErrorApi[]
}
+/**
+ * * `quick` - Read within a small budget, so the checks can run on every load
+ * * `full` - Read with the same limits as the insight's own query
+ */
+export type InsightDataChecksDepthEnumApi =
+ (typeof InsightDataChecksDepthEnumApi)[keyof typeof InsightDataChecksDepthEnumApi]
+
+export const InsightDataChecksDepthEnumApi = {
+ Quick: 'quick',
+ Full: 'full',
+} as const
+
export interface InsightDataChecksRequestApi {
/** The insight query to check, in the shape an insight stores it: an `InsightVizNode` with its `source`. Pass the query of a saved insight, or one that is not saved yet. */
query: _InsightQuerySchemaApi
@@ -9426,6 +9447,11 @@ export interface InsightDataChecksRequestApi {
* @maxLength 128
*/
client_query_id?: string
+ /** `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.
+ *
+ * * `quick` - Read within a small budget, so the checks can run on every load
+ * * `full` - Read with the same limits as the insight's own query */
+ depth?: InsightDataChecksDepthEnumApi
}
/**
@@ -9861,9 +9887,25 @@ export const InsightsActivityRetrieveFormat = {
} as const
export type InsightsDataChecksRetrieveParams = {
+ /**
+ * `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.
+ *
+ * * `quick` - Read within a small budget, so the checks can run on every load
+ * * `full` - Read with the same limits as the insight's own query
+ * @minLength 1
+ */
+ depth?: InsightsDataChecksRetrieveDepth
format?: InsightsDataChecksRetrieveFormat
}
+export type InsightsDataChecksRetrieveDepth =
+ (typeof InsightsDataChecksRetrieveDepth)[keyof typeof InsightsDataChecksRetrieveDepth]
+
+export const InsightsDataChecksRetrieveDepth = {
+ Quick: 'quick',
+ Full: 'full',
+} as const
+
export type InsightsDataChecksRetrieveFormat =
(typeof InsightsDataChecksRetrieveFormat)[keyof typeof InsightsDataChecksRetrieveFormat]
diff --git a/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsx b/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsx
index 6252c415a2de..3d1fdb3133b8 100644
--- a/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsx
+++ b/products/product_analytics/frontend/insights/dataChecks/DataChecksPopover.tsx
@@ -1,7 +1,17 @@
import { useActions, useMountedLogic, useValues } from 'kea'
import { useId } from 'react'
-import { Accordion, Button, Popover, PopoverContent, PopoverTrigger, Skeleton, Spinner, Text } from '@posthog/quill'
+import {
+ Accordion,
+ Button,
+ Dot,
+ Popover,
+ PopoverContent,
+ PopoverTrigger,
+ Skeleton,
+ Spinner,
+ Text,
+} from '@posthog/quill'
import { humanFriendlyNumber } from 'lib/utils/numbers'
import { pluralize } from 'lib/utils/strings'
@@ -25,6 +35,9 @@ function summary(status: InsightDataChecksStatus, coverage: InsightDataChecksCov
if (status === 'failed') {
return "The checks didn't finish. The chart's own numbers are not affected."
}
+ if (status === 'on_request') {
+ return "Checking this chart's events reads a lot of data, so the checks don't run on their own."
+ }
if (!coverage) {
return 'No check could run on this chart.'
}
@@ -46,8 +59,8 @@ export interface DataChecksPopoverProps {
export function DataChecksPopover({ insightProps, trigger, children }: DataChecksPopoverProps): JSX.Element {
const logic = insightDataChecksLogic(insightProps)
- const { visibleChecks, checkedQuery, result, status } = useValues(logic)
- const { loadDataChecks, reportButtonClicked, reportDataChecksViewed } = useActions(logic)
+ const { visibleChecks, checkedQuery, result, status, canRunFullCheck } = useValues(logic)
+ const { loadDataChecks, reportButtonClicked, reportDataChecksViewed, runFullCheck } = useActions(logic)
const { user } = useValues(userLogic)
const userId = user?.uuid ?? ''
// Mounted outside the popover content, so a half-written note survives the popover closing.
@@ -72,29 +85,36 @@ export function DataChecksPopover({ insightProps, trigger, children }: DataCheck
data-attr="insight-data-checks-list"
>
<Text id={titleId} size="sm" weight="medium">
- Checks we've run on this data
+ {status === 'on_request' ? 'Checks we can run on this data' : "Checks we've run on this data"}
</Text>
- {visibleChecks.length > 0 ? (
+ {/* Every check waits for the same request, so the reason is said once, below the list. */}
+ {visibleChecks.length > 0 && status !== 'on_request' ? (
<Accordion>
{visibleChecks.map((check) => (
<InsightDataCheckRow key={check.key} insightProps={insightProps} check={check} />
))}
</Accordion>
) : (
- status === 'checking' && (
+ (status === 'checking' || status === 'on_request') && (
<ul className="-mx-2 my-0 flex list-none flex-col p-0">
{Object.entries(DATA_CHECK_NAMES).map(([key, name]) => (
<li key={key} className="flex items-start gap-2 px-2 py-1">
<span className="flex h-5 w-4 shrink-0 items-center justify-center">
- <Spinner aria-hidden className="text-(--muted-foreground)" />
+ {status === 'checking' ? (
+ <Spinner aria-hidden className="text-(--muted-foreground)" />
+ ) : (
+ <Dot variant="default" />
+ )}
</span>
<span className="flex flex-col">
<Text size={null} weight="medium" render={<span />}>
{name}
</Text>
- <span className="flex h-4 items-center">
- <Skeleton className="h-2.5 w-28" />
- </span>
+ {status === 'checking' && (
+ <span className="flex h-4 items-center">
+ <Skeleton className="h-2.5 w-28" />
+ </span>
+ )}
</span>
</li>
))}
@@ -120,7 +140,18 @@ export function DataChecksPopover({ insightProps, trigger, children }: DataCheck
Try again
</Button>
)}
- {(status === 'passed' || status === 'warning' || status === 'problem') && (
+ {canRunFullCheck && (
+ <Button
+ variant="outline"
+ size="sm"
+ className="self-start"
+ onClick={() => checkedQuery && runFullCheck(checkedQuery)}
+ data-attr="insight-data-checks-run-full"
+ >
+ Run a full check
+ </Button>
+ )}
+ {(status === 'passed' || status === 'warning' || status === 'problem' || status === 'partial') && (
<DataChecksFeedback userId={userId} />
)}
</PopoverContent>
diff --git a/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx b/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx
index 7e8dded9d655..3a84382b7260 100644
--- a/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx
+++ b/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx
@@ -27,7 +27,10 @@ export function DataChecksTag({
<IconCheck key={status} />
) : status === 'checking' ? (
<Spinner key={status} textColored />
- ) : status === 'no_events' || status === 'inconclusive' ? (
+ ) : status === 'no_events' ||
+ status === 'inconclusive' ||
+ status === 'on_request' ||
+ status === 'partial' ? (
<IconCircleDashed key={status} />
) : (
<IconWarning key={status} />
diff --git a/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx b/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx
index f98b8dc94080..f21d32219489 100644
--- a/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx
+++ b/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx
@@ -19,6 +19,10 @@ function statusLabel(status: InsightDataChecksStatus, findingCount: number): str
return 'No data to check'
case 'inconclusive':
return 'Data not checked'
+ case 'on_request':
+ return 'Data not checked yet'
+ case 'partial':
+ return 'Data partly checked'
case 'warning':
return pluralize(findingCount, 'data warning')
case 'problem':
diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
index c55928284b1a..d806832dcdfe 100644
--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
@@ -56,6 +56,7 @@ const SKIP_REASONS: Record<InsightDataCheckSkipReasonEnumApi, string> = {
filtered_by_traffic_type: 'this insight already filters on traffic type',
hidden_property: 'a property it reads is hidden from you',
invalid_filter: "the project's internal and test users filter cannot run",
+ on_request: 'it would read a lot of data, so it runs only in a full check',
}
function eventCount(check: InsightDataCheckApi): string {
diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts
index e31db25ecfa3..b6e290073a48 100644
--- a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts
@@ -31,6 +31,7 @@ describe('insightDataChecksLogic', () => {
const dataChecks = jest.fn()
const cancelledQueryIds: string[] = []
let sentSources: Record<string, unknown>[]
+ let sentDepths: (string | undefined)[]
let savedFilters: unknown[]
let capture: jest.SpyInstance
@@ -42,8 +43,12 @@ describe('insightDataChecksLogic', () => {
const answer =
(response: [number, Record<string, unknown>], held?: Promise<void>, onSent?: () => void) =>
async ({ request }: { request: Request }): Promise<[number, Record<string, unknown>]> => {
- const body = (await request.json()) as { query: { source: Record<string, unknown> } }
+ const body = (await request.json()) as {
+ query: { source: Record<string, unknown> }
+ depth?: string
+ }
sentSources.push(body.query.source)
+ sentDepths.push(body.depth)
onSent?.()
await held
return response
@@ -59,6 +64,7 @@ describe('insightDataChecksLogic', () => {
dataChecks.mockReset()
capture = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined)
sentSources = []
+ sentDepths = []
savedFilters = []
cancelledQueryIds.length = 0
useMocks({
@@ -224,6 +230,37 @@ describe('insightDataChecksLogic', () => {
expect(dataChecks).toHaveBeenCalledTimes(requests)
})
+ it('runs a full check only after a person asks, and only for that query', async () => {
+ dataChecks.mockImplementationOnce(answer([200, checksOf('on_request')]))
+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))
+ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()
+ expect(logic.values.canRunFullCheck).toBe(true)
+
+ await expectLogic(logic, () => {
+ logic.actions.runFullCheck(logic.values.checkedQuery!)
+ }).toFinishAllListeners()
+ expect(logic.values).toMatchObject({ status: 'passed', canRunFullCheck: false })
+
+ vizData.actions.updateQuerySource({ dateRange: { date_from: '-30d' } })
+ await expectLogic(logic, () => logic.actions.checkData()).toFinishAllListeners()
+
+ expect(sentDepths).toEqual(['quick', 'full', 'quick'])
+ })
+
+ it.each([
+ ['a check stopped at a limit', 'partial', 'read_limit', true],
+ ['every check stopped at a limit', 'inconclusive', 'time_limit', true],
+ ['another check found a problem', 'problem', 'memory_limit', true],
+ ['the only skipped check could not run for another reason', 'partial', 'hidden_property', false],
+ ])('offers a full check of a partly checked chart when %s', async (_name, status, skipReason, offered) => {
+ const skipped = { key: 'internal_users', level: 'skipped', skip_reason: skipReason }
+ dataChecks.mockImplementation(answer([200, { ...checksOf(status), checks: [skipped] }]))
+
+ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()
+
+ expect(logic.values.canRunFullCheck).toBe(offered)
+ })
+
it.each([
['hides the mark from a person who may not run the checks', 403, 'unavailable'],
['reports a check the server could not finish', 500, 'failed'],
diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts
index 0f2f2f86c5fa..450d5b583d8d 100644
--- a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts
@@ -44,6 +44,7 @@ import { insightsDataChecksCreate } from 'products/product_analytics/frontend/ge
import type {
InsightDataCheckApi,
InsightDataCheckKeyEnumApi,
+ InsightDataCheckSkipReasonEnumApi,
InsightDataChecksApi,
InsightDataChecksRequestApi,
InsightDataChecksStatusEnumApi,
@@ -110,6 +111,7 @@ export interface insightDataChecksLogicValues {
sidePanelOpen: boolean // sidePanelStateLogic
currentTeam: TeamPublicType | TeamType | null // teamLogic
currentTeamLoading: boolean // teamLogic
+ canRunFullCheck: boolean
checkAgainRequested: boolean
checkedQuery: DataChecksQuery | null
checks: InsightDataCheckApi[]
@@ -118,6 +120,7 @@ export interface insightDataChecksLogicValues {
filterRemovesBots: boolean
filtersTestAccounts: boolean
findings: InsightDataCheckApi[]
+ fullCheckQuery: DataChecksQuery | null
labelHeldOpen: boolean
labelOpen: boolean
lastResult: InsightDataChecksApi | null
@@ -258,6 +261,9 @@ export interface insightDataChecksLogicActions {
reportDataChecksViewed: () => {
value: true
}
+ runFullCheck: (query: DataChecksQuery) => {
+ query: DataChecksQuery
+ }
setFiltersTestAccounts: (
filtersTestAccounts: boolean,
check: InsightDataCheckKeyEnumApi
@@ -294,6 +300,12 @@ export interface insightDataChecksLogicMeta {
queryFiltersTestAccounts: (checkedQuery: DataChecksQuery | null) => boolean
checks: (result: InsightDataChecksApi | null) => InsightDataCheckApi[]
findings: (checks: InsightDataCheckApi[]) => InsightDataCheckApi[]
+ canRunFullCheck: (
+ status: InsightDataChecksStatus,
+ checks: InsightDataCheckApi[],
+ checkedQuery: DataChecksQuery | null,
+ fullCheckQuery: DataChecksQuery | null
+ ) => boolean
labelOpen: (status: InsightDataChecksStatus, labelHeldOpen: boolean) => boolean
}
}
@@ -306,7 +318,7 @@ export type insightDataChecksLogicType = MakeLogicType<
>
const FORCED_REFRESHES: (RefreshType | undefined)[] = ['force_async', 'force_blocking']
-// The checks run up to four queries of 20 seconds each, after a wait for server slots.
+// A full check stops its queries after 60 seconds on the server, which leaves the rest for a wait for server slots.
const DATA_CHECKS_TIMEOUT_MS = 120_000
const LABEL_OPEN_MS = 2500
const YIELDED_TO_CHART = 'yielded to chart'
@@ -319,6 +331,13 @@ function readsEvents(series: Node): boolean {
return isEventsOrActionsNode(series) || (isGroupNode(series) && series.nodes.every(isEventsOrActionsNode))
}
+// A full check lifts these limits. It still does not count a day with too many events, so that reason is not here.
+const LIMIT_SKIP_REASONS: (InsightDataCheckSkipReasonEnumApi | null)[] = ['read_limit', 'time_limit', 'memory_limit']
+
+function stoppedAtALimit(skipReason: InsightDataCheckSkipReasonEnumApi | null): boolean {
+ return LIMIT_SKIP_REASONS.includes(skipReason)
+}
+
export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> = kea<insightDataChecksLogicType>([
props({} as InsightLogicProps),
key(keyForInsightLogicProps('new')),
@@ -365,8 +384,9 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
}),
reportCheckExpanded: (check: InsightDataCheckApi) => ({ check }),
reportDataChecksViewed: true,
+ runFullCheck: (query: DataChecksQuery) => ({ query }),
}),
- loaders(({ cache }) => ({
+ loaders(({ cache, values }) => ({
dataChecksRun: [
null as DataChecksRun | null,
{
@@ -395,6 +415,7 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
{
query,
client_query_id: clientQueryId,
+ depth: values.fullCheckQuery === query ? 'full' : 'quick',
} as unknown as InsightDataChecksRequestApi,
undefined,
{ signal: abortController.signal }
@@ -426,6 +447,13 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
loadDataChecks: (_, query) => query,
},
],
+ // A person asks for each query, so a changed query gets a quick check again.
+ fullCheckQuery: [
+ null as DataChecksQuery | null,
+ {
+ runFullCheck: (_, { query }) => query,
+ },
+ ],
// A person can refresh the chart during a run. That run is for the events as they were before.
checkAgainRequested: [
false,
@@ -526,6 +554,18 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
(s) => [s.checks],
(checks: InsightDataCheckApi[]): InsightDataCheckApi[] => checks.filter(isFinding),
],
+ canRunFullCheck: [
+ (s) => [s.status, s.checks, s.checkedQuery, s.fullCheckQuery],
+ (
+ status: InsightDataChecksStatus,
+ checks: InsightDataCheckApi[],
+ checkedQuery: DataChecksQuery | null,
+ fullCheckQuery: DataChecksQuery | null
+ ): boolean =>
+ checkedQuery !== null &&
+ checkedQuery !== fullCheckQuery &&
+ (status === 'on_request' || checks.some((check) => stoppedAtALimit(check.skip_reason))),
+ ],
labelOpen: [
(s) => [s.status, s.labelHeldOpen],
// A finding keeps its label, because the icon alone does not say what the person needs to look at.
@@ -580,6 +620,13 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
loadDataSuccess: ({ payload }) => {
actions.checkData(FORCED_REFRESHES.includes(payload?.refresh))
},
+ runFullCheck: ({ query }) => {
+ // pinned: analytics event name, renaming breaks dashboards
+ posthog.capture('insight data checks full check requested', { status: values.status })
+ if (!values.dataChecksRunLoading) {
+ actions.loadDataChecks(query)
+ }
+ },
setFiltersTestAccounts: ({ filtersTestAccounts, check }) => {
actions.updateQuerySource({ filterTestAccounts: filtersTestAccounts })
if (filtersTestAccounts) {
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 420df5f729fb..72340f0c20a1 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -62028,6 +62028,7 @@ export namespace Schemas {
* * `filtered_by_traffic_type` - The insight already filters on traffic type
* * `hidden_property` - A property the check reads is hidden from you
* * `invalid_filter` - The project's internal and test users filter cannot run
+ * * `on_request` - The check would read a lot of data, so it runs only on request
*/
export type InsightDataCheckSkipReasonEnum = typeof InsightDataCheckSkipReasonEnum[keyof typeof InsightDataCheckSkipReasonEnum];
@@ -62041,6 +62042,7 @@ export namespace Schemas {
FilteredByTrafficType: 'filtered_by_traffic_type',
HiddenProperty: 'hidden_property',
InvalidFilter: 'invalid_filter',
+ OnRequest: 'on_request',
} as const;
/**
@@ -62102,7 +62104,8 @@ export namespace Schemas {
* * `too_few_sampled_events` - The sample has too few events to give a share
* * `filtered_by_traffic_type` - The insight already filters on traffic type
* * `hidden_property` - A property the check reads is hidden from you
- * * `invalid_filter` - The project's internal and test users filter cannot run */
+ * * `invalid_filter` - The project's internal and test users filter cannot run
+ * * `on_request` - The check would read a lot of data, so it runs only on request */
skip_reason: InsightDataCheckSkipReasonEnum | null;
/** `counted` when the check counted every event on the days checked, `sampled` when it read a sample. Null when the check did not run.
*
@@ -62167,6 +62170,8 @@ export namespace Schemas {
* * `passed` - No data issues found
* * `warning` - A check found a small issue
* * `problem` - A check found a large issue
+ * * `partial` - Some checks could not run, and the others found no issue
+ * * `on_request` - The checks would read a lot of data, so they run only on request
* * `no_events` - No events to check
* * `unsupported` - This kind of insight cannot be checked
* * `inconclusive` - No check could run
@@ -62178,6 +62183,8 @@ export namespace Schemas {
Passed: 'passed',
Warning: 'warning',
Problem: 'problem',
+ Partial: 'partial',
+ OnRequest: 'on_request',
NoEvents: 'no_events',
Unsupported: 'unsupported',
Inconclusive: 'inconclusive',
@@ -62217,11 +62224,13 @@ export namespace Schemas {
}
export interface InsightDataChecks {
- /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run.
+ /** The worst result across the checks. `no_events` when no event matches the insight's events, filters and date range. `unsupported` when the insight has no event series to check, such as a SQL insight, and when a property the checks read is hidden from the caller. `inconclusive` when no check could run. `partial` when some checks could not run and the others found nothing. `on_request` when the checks would read a lot of data for a quick check, so they did not run: send `depth` `full` to run them.
*
* * `passed` - No data issues found
* * `warning` - A check found a small issue
* * `problem` - A check found a large issue
+ * * `partial` - Some checks could not run, and the others found no issue
+ * * `on_request` - The checks would read a lot of data, so they run only on request
* * `no_events` - No events to check
* * `unsupported` - This kind of insight cannot be checked
* * `inconclusive` - No check could run */
@@ -62232,6 +62241,18 @@ export namespace Schemas {
coverage: InsightDataChecksCoverage | null;
}
+ /**
+ * * `quick` - Read within a small budget, so the checks can run on every load
+ * * `full` - Read with the same limits as the insight's own query
+ */
+ export type InsightDataChecksDepthEnum = typeof InsightDataChecksDepthEnum[keyof typeof InsightDataChecksDepthEnum];
+
+
+ export const InsightDataChecksDepthEnum = {
+ Quick: 'quick',
+ Full: 'full',
+ } as const;
+
export interface InsightDataChecksRequest {
/** The insight query to check, in the shape an insight stores it: an `InsightVizNode` with its `source`. Pass the query of a saved insight, or one that is not saved yet. */
query: _InsightQuerySchema;
@@ -62240,6 +62261,11 @@ export namespace Schemas {
* @maxLength 128
*/
client_query_id?: string;
+ /** `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.
+ *
+ * * `quick` - Read within a small budget, so the checks can run on every load
+ * * `full` - Read with the same limits as the insight's own query */
+ depth?: InsightDataChecksDepthEnum;
}
export interface InsightPlatformAlertCondition {
@@ -131215,9 +131241,25 @@ export namespace Schemas {
} as const;
export type InsightsDataChecksRetrieveParams = {
+ /**
+ * `quick`, the default, reads within a small budget so the checks can run on every load. On an insight they would read a lot of data for, a quick check does not run and `status` is `on_request`. `full` runs the checks with the same limits as the insight's own query.
+ *
+ * * `quick` - Read within a small budget, so the checks can run on every load
+ * * `full` - Read with the same limits as the insight's own query
+ * @minLength 1
+ */
+ depth?: InsightsDataChecksRetrieveDepth;
format?: InsightsDataChecksRetrieveFormat;
};
+ export type InsightsDataChecksRetrieveDepth = typeof InsightsDataChecksRetrieveDepth[keyof typeof InsightsDataChecksRetrieveDepth];
+
+
+ export const InsightsDataChecksRetrieveDepth = {
+ Quick: 'quick',
+ Full: 'full',
+ } as const;
+
export type InsightsDataChecksRetrieveFormat = typeof InsightsDataChecksRetrieveFormat[keyof typeof InsightsDataChecksRetrieveFormat];
diff --git a/services/mcp/src/tools/insights/trustCheck.ts b/services/mcp/src/tools/insights/trustCheck.ts
index 9d477e4ac81b..ae603ee8f0ff 100644
--- a/services/mcp/src/tools/insights/trustCheck.ts
+++ b/services/mcp/src/tools/insights/trustCheck.ts
@@ -35,7 +35,11 @@ type Result = WithAgentNote<Schemas.InsightDataChecks>
const AGENT_NOTE =
'When a check has the level `warning` or `problem`, tell the user what it found in one sentence, with its share ' +
"of the events, and offer the fix. Do not change the insight, the query or the project's filter settings before " +
- 'the user agrees. When every check passed, say so in a few words. Do not list the checks that passed.'
+ 'the user agrees. When every check passed, say so in a few words. Do not list the checks that passed. When the ' +
+ '`status` is `partial`, say which checks did not run and give each `skip_reason` in plain words.'
+
+// An agent's call is already a request to check, so it runs the full check, whatever it reads.
+const FULL_DEPTH = { depth: 'full' }
const insightTrustCheck = (): ToolBase<typeof schema, Result> => ({
name: 'insight-trust-check',
@@ -48,11 +52,12 @@ const insightTrustCheck = (): ToolBase<typeof schema, Result> => ({
? await context.api.request<Schemas.InsightDataChecks>({
method: 'GET',
path: `/api/projects/${projectId}/insights/${encodeURIComponent(String(id))}/data_checks/`,
+ query: FULL_DEPTH,
})
: await context.api.request<Schemas.InsightDataChecks>({
method: 'POST',
path: `/api/projects/${projectId}/insights/data_checks/`,
- body: { query },
+ body: { query, ...FULL_DEPTH },
})
return withAgentNote(result, AGENT_NOTE)
},