:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(insights): suggest filter rules from insight data checks (#120060)
Sam Penningtone6ff5fc97f23
diff --git a/products/product_analytics/backend/facade/contracts.py b/products/product_analytics/backend/facade/contracts.pyindex 06eb92558b55..061313326cef 100644--- a/products/product_analytics/backend/facade/contracts.py+++ b/products/product_analytics/backend/facade/contracts.py@@ -61,6 +61,16 @@ class SavedInsightDefinition:query: dict[str, Any]+@dataclass(frozen=True)+class InsightDataCheckFilter:+ """A rule in the shape the project's internal and test users filter stores it."""++ key: str+ type: str+ operator: str+ value: list[str]++@dataclass(frozen=True)class InsightDataCheck:key: InsightDataCheckKey@@ -75,7 +85,15 @@ class InsightDataCheck:# The most affected values first, so a finding says what is affected as well as how much.top_events: list[str] = field(default_factory=list)top_hosts: list[str] = field(default_factory=list)+ distinct_hosts: int | None = Nonetop_bots: list[str] = field(default_factory=list)+ distinct_bots: int | None = None+ # The internal users check's `events`, split into what the project's filter removes and what it keeps.+ removed_by_filter: int | None = None+ missed_by_filter: int | None = None+ top_emails: list[str] = field(default_factory=list)+ distinct_emails: int | None = None+ filter_to_add: InsightDataCheckFilter | None = None@dataclass(frozen=True)@@ -113,6 +131,8 @@ class TeamEmailAccess:# A match reveals that an email belongs to the team, so each source needs the caller to be allowed to read it.member_emails: boolverified_domains: bool+ # The emails a check finds are person data, so the check names them only for a caller who may read persons.+ person_emails: bool@dataclass(frozen=True)diff --git a/products/product_analytics/backend/facade/data_checks.py b/products/product_analytics/backend/facade/data_checks.pyindex 1166081c5978..7f1985a3490f 100644--- a/products/product_analytics/backend/facade/data_checks.py+++ b/products/product_analytics/backend/facade/data_checks.py@@ -21,6 +21,7 @@ def run_insight_data_checks("""Check the events an insight query reads for internal users, local traffic, bots and events sent twice.The checks read the events as `user`. They do not run when a property they read is hidden from that person.- The internal users check matches only the member emails and verified domains that `email_access` allows.+ The internal users check matches only the member emails and verified domains that `email_access` allows, and+ names the emails it finds only when `email_access` allows person emails."""return insight_data_checks.run_insight_data_checks(query, team, user, email_access=email_access)diff --git a/products/product_analytics/backend/insight_data_check_results.py b/products/product_analytics/backend/insight_data_check_results.pyindex c9fbbde6fde9..6666e1d023c0 100644--- a/products/product_analytics/backend/insight_data_check_results.py+++ b/products/product_analytics/backend/insight_data_check_results.py@@ -2,7 +2,7 @@from posthog.dataclasses import frozen-from products.product_analytics.backend.facade.contracts import InsightDataCheck+from products.product_analytics.backend.facade.contracts import InsightDataCheck, InsightDataCheckFilterfrom products.product_analytics.backend.facade.enums import (InsightDataCheckAction,InsightDataCheckKey,@@ -20,6 +20,11 @@ MIN_DUPLICATE_EVENTS = 100MIN_DUPLICATE_SHARE = 0.05PROBLEM_SHARE = 0.2+# An event with no user agent has the type "Automation", and server-side SDKs send those, so it is not a crawler.+CRAWLER_TRAFFIC_TYPES = ("Bot", "AI Agent")++LOCAL_HOST_PATTERN = r"(?i)^(localhost|[a-z0-9.-]+\.localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\])(:[0-9]+)?$"+@frozenclass Skipped:@@ -28,9 +33,13 @@ class Skipped:@frozenclass LocalEvents:+ # The events from a local host that the project's internal and test users filter keeps.count: int+ distinct_hosts: inttop_hosts: list[str]top_events: list[str]+ # Every host the check found, up to a limit, so the rule to add names only what the person saw.+ hosts: list[str]@frozen@@ -40,6 +49,9 @@ class InternalEvents:# Events from the team's own emails or test accounts that the filter keeps.missed_by_filter: inttop_events: list[str]+ distinct_missed_emails: int+ top_missed_emails: list[str]+ missed_emails: list[str]@propertydef count(self) -> int:@@ -50,7 +62,6 @@ class InternalEvents:class CountedEvents:# Every event the insight reads on the days checked, so a counted check needs no sample for its share.events: int- local: LocalEventsinternal: InternalEvents | Skipped@@ -60,6 +71,7 @@ class SampledEvents:events: intdistinct_ids: intbot_events: int+ distinct_bots: inttop_bots: list[str]top_bot_events: list[str]repeated_events: int@@ -73,10 +85,24 @@ class SampledEvents:@frozenclass CheckedEvents:counted: CountedEvents | Skipped+ local: LocalEvents | Skippedsample: SampledEvents | Skippedinsight_filters_traffic_type: bool = False+def events_on_checked_days(checked: CheckedEvents) -> int | None:+ if isinstance(checked.counted, CountedEvents):+ return checked.counted.events+ if isinstance(checked.sample, SampledEvents):+ return round(checked.sample.events / checked.sample.rate)+ return None+++def _excluding(key: str, property_type: str, values: list[str]) -> InsightDataCheckFilter:+ # Names only the values the check found, because a broader rule could remove real users the person never saw.+ return InsightDataCheckFilter(key=key, type=property_type, operator="is_not", value=sorted(values))++def not_checked(key: InsightDataCheckKey, reason: InsightDataCheckSkipReason | None = None) -> InsightDataCheck:return InsightDataCheck(key=key, level=InsightDataCheckLevel.SKIPPED, skip_reason=reason)@@ -126,28 +152,37 @@ def _internal_users_check(checked: CheckedEvents) -> InsightDataCheck:share=share,action=action if is_finding else None,top_events=internal.top_events,+ removed_by_filter=internal.removed_by_filter,+ missed_by_filter=internal.missed_by_filter,+ top_emails=internal.top_missed_emails,+ distinct_emails=internal.distinct_missed_emails,+ filter_to_add=_excluding("email", "person", internal.missed_emails) if internal.missed_emails else None,)def _local_development_check(checked: CheckedEvents) -> InsightDataCheck:key = InsightDataCheckKey.LOCAL_DEVELOPMENT- counted = checked.counted- if isinstance(counted, Skipped):- return not_checked(key, counted.reason)+ local = checked.local+ if isinstance(local, Skipped):+ return not_checked(key, local.reason)- local = counted.local- share = _share(local.count, counted.events)+ events = events_on_checked_days(checked)+ share = _share(local.count, events) if events is not None else None# No minimum share: a local session also creates test persons and flag exposures, whatever its share.is_finding = local.count >= MIN_FINDING_EVENTSreturn InsightDataCheck(key=key,- level=_finding_level(share) if is_finding else InsightDataCheckLevel.PASSED,+ # Never a problem: a project can serve real users from a local host, such as an app in a native wrapper,+ # so the person who owns the project decides whether to keep these events.+ level=InsightDataCheckLevel.WARNING if is_finding else InsightDataCheckLevel.PASSED,method=InsightDataCheckMethod.COUNTED,events=local.count,share=share,action=InsightDataCheckAction.OPEN_FILTER_SETTINGS if is_finding else None,top_events=local.top_events,top_hosts=local.top_hosts,+ distinct_hosts=local.distinct_hosts,+ filter_to_add=_excluding("$host", "event", local.hosts) if is_finding else None,)@@ -174,6 +209,8 @@ def _bot_traffic_check(checked: CheckedEvents) -> InsightDataCheck:action=InsightDataCheckAction.OPEN_FILTER_SETTINGS if is_finding else None,top_events=sample.top_bot_events,top_bots=sample.top_bots,+ distinct_bots=sample.distinct_bots,+ filter_to_add=_excluding("$virt_traffic_type", "event", list(CRAWLER_TRAFFIC_TYPES)) if is_finding else None,)diff --git a/products/product_analytics/backend/insight_data_checks.py b/products/product_analytics/backend/insight_data_checks.pyindex 6364e44c15f7..4531a4201139 100644--- a/products/product_analytics/backend/insight_data_checks.py+++ b/products/product_analytics/backend/insight_data_checks.py@@ -48,6 +48,8 @@ from products.product_analytics.backend.facade.enums import ()from products.product_analytics.backend.insight_data_check_days import CheckWindow, days_between, newest_days_withinfrom products.product_analytics.backend.insight_data_check_results import (+ CRAWLER_TRAFFIC_TYPES,+ LOCAL_HOST_PATTERN,CheckedEvents,CountedEvents,InternalEvents,@@ -55,6 +57,7 @@ from products.product_analytics.backend.insight_data_check_results import (SampledEvents,Skipped,evaluate_data_checks,+ events_on_checked_days,found_no_events,not_checked,status_of,@@ -78,9 +81,6 @@ MAX_BYTES_READ_PER_SHARD = 1_000_000_000# The sample reads wide columns and sorts them for the repeat window, so it gets a tighter limit.MAX_SAMPLE_BYTES_READ_PER_SHARD = 250_000_000-LOCAL_HOST_PATTERN = r"(?i)^(localhost|[a-z0-9.-]+\.localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\])(:[0-9]+)?$"-# An event with no user agent has the type "Automation", and server-side SDKs send those, so it is not a crawler.-CRAWLER_TRAFFIC_TYPES = ("Bot", "AI Agent")# A person cannot repeat an action this fast, but an app that sends an event twice does.REPEAT_WINDOW_MILLISECONDS = 100# A hidden property reads as empty, so the checks cannot run when one of these is hidden.@@ -102,24 +102,40 @@ _EVENTS_PER_DAY_QUERY = """GROUP BY day"""-# One scan counts both checks, so the internal users check costs little more than the local check alone.-# HogQL has no topKIf, and topK skips NULL, so each list ranks only the events its check found.+# HogQL has no topKIf, and topK skips NULL, so the list ranks only the events the check found._COUNTED_EVENTS_QUERY = """SELECTcount() AS events,- countIf({is_local_host}) AS local_events,- topK(3)(if({is_local_host}, toString(properties.$host), NULL)) AS top_local_hosts,- topK(3)(if({is_local_host}, event, NULL)) AS top_local_events,countIf(NOT {passes_filter}) AS removed_by_filter,countIf({passes_filter} AND {is_team_email}) AS missed_by_filter,- topK(3)(if(NOT {passes_filter} OR {is_team_email}, event, NULL)) AS top_internal_events+ topK(3)(if(NOT {passes_filter} OR {is_team_email}, event, NULL)) AS top_internal_events,+ uniqIf({email}, {passes_filter} AND {is_team_email}) AS distinct_missed_emails,+ topK(3)(if({passes_filter} AND {is_team_email}, {email}, NULL)) AS top_missed_emails,+ groupUniqArrayIf(50)({stored_email}, {passes_filter} AND {is_team_email}) AS missed_emails+ FROM events+"""++# The host match goes in WHERE, and ClickHouse moves it to PREWHERE itself when `$host` has its own column.+# An explicit PREWHERE fails when a hidden property wraps `properties` in the JSONDropKeys UDF, because ClickHouse+# does not resolve executable UDFs in PREWHERE.+_LOCAL_EVENTS_QUERY = """+ SELECT+ count() AS events,+ uniqExact(toString(properties.$host)) AS distinct_hosts,+ topK(3)(toString(properties.$host)) AS top_hosts,+ topK(3)(event) AS top_events,+ groupUniqArray(50)(toString(properties.$host)) AS hostsFROM events"""_IS_LOCAL_HOST = "match(toString(properties.$host), {local_host_pattern})"-# Without persons on events, `person` is an inner join that drops events with no person, so read the event's copy.-_IS_TEAM_EMAIL = "match(lower(toString(poe.properties.email)), {team_email_pattern})"+# The suggested rule is a person rule, so the check reads the email from the same person, which is the event's copy+# or the current person depending on the persons-on-events mode.+# An `is_not` rule compares case-sensitively, so the suggested rule names each email as it is stored.+_STORED_EMAIL = "toString(person.properties.email)"+_EMAIL = "lower({stored_email})"+_IS_TEAM_EMAIL = "match({email}, {team_email_pattern})"_SAMPLED_EVENTS_QUERY = """SELECT@@ -244,7 +260,9 @@ def _filters_on_traffic_type(where: ast.Expr) -> bool:def _check_modifiers(chart_modifiers: HogQLQueryModifiers, *, keep_project_bot_rules: bool) -> HogQLQueryModifiers:# With `debug` on, a failed query returns an error and no rows, which would read as no events.- update: dict[str, Any] = {"debug": False}+ # Without the pushdown, a persons join reads the latest version of every person in the project, however few+ # events the check reads.+ update: dict[str, Any] = {"debug": False, "personIdPushdown": True}if not keep_project_bot_rules:# The bot check uses PostHog's definition of a bot, so a project's own bot rules do not apply.update["customBotDefinitions"] = []@@ -292,6 +310,13 @@ def _top_three(values_by_key: tuple[list[str], list[int]] | None) -> list[str]:return [key for key, value in ranked if key and value > 0][:3]+def _distinct_keys(values_by_key: tuple[list[str], list[int]] | None) -> int:+ if not values_by_key:+ return 0+ keys, values = values_by_key+ return sum(1 for key, value in zip(keys, values) if key and value > 0)++def _count_events_per_day(events: ChartEvents, reader: _EventReader, days: list[date], timezone: ZoneInfo) -> dict[date, int] | Skipped:@@ -329,13 +354,19 @@ def _choose_window(events: ChartEvents, reader: _EventReader, timezone: ZoneInfo@frozenclass _InternalUsersMatch:passes_filter: ast.Expr+ email: ast.Expr+ stored_email: ast.Expris_team_email: ast.Expr- # The person fields `passes_filter` reads from the current person join, or None when it needs no join.+ # The person fields the match reads from the current person join, or None when it needs no join.person_fields: frozenset[str] | None = None+ names_emails: bool = False_NO_INTERNAL_USERS = _InternalUsersMatch(- passes_filter=ast.Constant(value=True), is_team_email=ast.Constant(value=False)+ passes_filter=ast.Constant(value=True),+ email=ast.Constant(value=None),+ stored_email=ast.Constant(value=None),+ is_team_email=ast.Constant(value=False),)@@ -383,19 +414,27 @@ class _PersonFromLeftJoin(CloningVisitor):return super().visit_field(node)+def _read_person_keeping_personless_events(+ events: ChartEvents, expr: ast.Expr+) -> tuple[ast.Expr, frozenset[str] | None]:+ # Without persons on events, the person comes from an inner join that drops the events with no person, which+ # the insight counts. A left join on the same table reads the same person and keeps those events.+ if events.modifiers.personsOnEventsMode != PersonsOnEventsMode.DISABLED:+ return expr, None+ person_from_left_join = _PersonFromLeftJoin()+ expr = person_from_left_join.visit(expr)+ if person_from_left_join.person_fields is None:+ return expr, None+ return expr, frozenset(person_from_left_join.person_fields)++def _passes_internal_users_filter(events: ChartEvents, team: Team) -> tuple[ast.Expr, frozenset[str] | None]:# An insight that applies the filter has already dropped the events it removes.if events.filters_test_accounts or not team.test_account_filters:return ast.Constant(value=True), None- project_filter = property_to_expr(team.test_account_filters, team)- person_fields: frozenset[str] | None = None- # Without persons on events, the person comes from an inner join that drops the events with no person, which- # the insight counts. A left join on the same table reads the same person and keeps those events.- if events.modifiers.personsOnEventsMode == PersonsOnEventsMode.DISABLED:- person_from_left_join = _PersonFromLeftJoin()- project_filter = person_from_left_join.visit(project_filter)- if person_from_left_join.person_fields is not None:- person_fields = frozenset(person_from_left_join.person_fields)+ project_filter, person_fields = _read_person_keeping_personless_events(+ events, property_to_expr(team.test_account_filters, team)+ )# The insight's own WHERE drops an event whose filter is NULL, so NULL counts as removed.return ast.Call(name="ifNull", args=[project_filter, ast.Constant(value=False)]), person_fields@@ -451,12 +490,22 @@ def _internal_users_match(return Skipped(reason=InsightDataCheckSkipReason.HIDDEN_PROPERTY)team_emails = team_emails_of(reader.team, reader.user, email_access)- passes_filter, person_fields = _passes_internal_users_filter(events, reader.team)+ passes_filter, filter_person_fields = _passes_internal_users_filter(events, reader.team)+ stored_email, email_person_fields = _read_person_keeping_personless_events(events, parse_expr(_STORED_EMAIL))+ person_fields: frozenset[str] | None = None+ if filter_person_fields is not None or email_person_fields is not None:+ person_fields = (filter_person_fields or frozenset()) | (email_person_fields or frozenset())++ email = parse_expr(_EMAIL, placeholders={"stored_email": stored_email})return _InternalUsersMatch(passes_filter=passes_filter,person_fields=person_fields,+ names_emails=email_access.person_emails,+ email=email,+ stored_email=stored_email,is_team_email=parse_expr(- _IS_TEAM_EMAIL, placeholders={"team_email_pattern": ast.Constant(value=team_email_pattern(team_emails))}+ _IS_TEAM_EMAIL,+ placeholders={"email": email, "team_email_pattern": ast.Constant(value=team_email_pattern(team_emails))},),)@@ -468,10 +517,9 @@ def _count_every_event(query = events.select(_COUNTED_EVENTS_QUERY,where=where,- is_local_host=parse_expr(- _IS_LOCAL_HOST, placeholders={"local_host_pattern": ast.Constant(value=LOCAL_HOST_PATTERN)}- ),passes_filter=match.passes_filter,+ email=match.email,+ stored_email=match.stored_email,is_team_email=match.is_team_email,)if match.person_fields is not None:@@ -482,24 +530,67 @@ def _count_every_event(if isinstance(rows, Skipped):return rows- row: list[Any] = rows[0] if rows else [0, 0, [], [], 0, 0, []]- count, local_count, top_local_hosts, top_local_events, removed, missed, top_internal_events = row+ row: list[Any] = rows[0] if rows else [0, 0, 0, [], 0, [], []]+ count, removed, missed, top_internal_events, distinct_missed_emails, top_missed_emails, missed_emails = rowinternal: InternalEvents | Skippedif isinstance(internal_users, Skipped):internal = internal_userselse:internal = InternalEvents(- removed_by_filter=int(removed or 0), missed_by_filter=int(missed or 0), top_events=list(top_internal_events)+ removed_by_filter=int(removed or 0),+ missed_by_filter=int(missed or 0),+ top_events=list(top_internal_events),+ distinct_missed_emails=int(distinct_missed_emails or 0),+ top_missed_emails=list(top_missed_emails) if internal_users.names_emails else [],+ missed_emails=[email for email in missed_emails if email] if internal_users.names_emails else [],)- return CountedEvents(- events=int(count or 0),- local=LocalEvents(- count=int(local_count or 0), top_hosts=list(top_local_hosts), top_events=list(top_local_events)- ),- internal=internal,+ return CountedEvents(events=int(count or 0), internal=internal)+++def _count_local_events_the_filter_keeps(+ events: ChartEvents,+ reader: _EventReader,+ where: ast.Expr,+ passes_filter: ast.Expr,+ person_fields: frozenset[str] | None,+) -> LocalEvents | Skipped:+ is_local_host = parse_expr(+ _IS_LOCAL_HOST, placeholders={"local_host_pattern": ast.Constant(value=LOCAL_HOST_PATTERN)}+ )+ query = events.select(+ _LOCAL_EVENTS_QUERY,+ where=ast.And(exprs=[clone_expr(is_local_host), where, passes_filter]),+ )+ if person_fields is not None:+ local_events = ast.And(exprs=[clone_expr(where), clone_expr(is_local_host)])+ cast(ast.JoinExpr, query.select_from).next_join = _current_person_join(events, local_events, person_fields)+ rows = reader.rows_within_limits(+ query, query_type="insight_data_checks_local", max_bytes_read_per_shard=MAX_BYTES_READ_PER_SHARD+ )+ if isinstance(rows, Skipped):+ return rows++ row: list[Any] = rows[0] if rows else [0, 0, [], [], []]+ count, distinct_hosts, top_hosts, top_events, hosts = row+ return LocalEvents(+ count=int(count or 0),+ distinct_hosts=int(distinct_hosts or 0),+ top_hosts=list(top_hosts),+ top_events=list(top_events),+ hosts=list(hosts),)+def _count_local_events(events: ChartEvents, reader: _EventReader, where: ast.Expr) -> LocalEvents | Skipped:+ try:+ passes_filter, person_fields = _passes_internal_users_filter(events, reader.team)+ return _count_local_events_the_filter_keeps(events, reader, where, passes_filter, person_fields)+ except (ExposedHogQLError, ResolutionError, ExposedCHQueryError) as error:+ # The internal users check reports the broken filter, so this check counts every local event.+ capture_exception(error)+ return _count_local_events_the_filter_keeps(events, reader, where, ast.Constant(value=True), None)++def _count_events_on_checked_days(events: ChartEvents,reader: _EventReader,@@ -551,6 +642,7 @@ def _sample_events(events=int(sampled or 0),distinct_ids=int(distinct_ids or 0),bot_events=int(bot_events or 0),+ distinct_bots=_distinct_keys(by_bot),top_bots=_top_three(by_bot),top_bot_events=_top_three(bots_by_event),repeated_events=int(repeated_events or 0),@@ -577,8 +669,9 @@ def _read_checked_days(counted = _count_events_on_checked_days(events, reader, where, email_access=email_access, hidden_properties=hidden_properties)+ local = _count_local_events(events, reader, where)sample = _sample_events(events, reader, where, events_before_filters=window.events_on_days_to_check)- return CheckedEvents(counted=counted, sample=sample, insight_filters_traffic_type=filters_traffic_type)+ return CheckedEvents(counted=counted, local=local, sample=sample, insight_filters_traffic_type=filters_traffic_type)def _sample_summary(sample: SampledEvents | Skipped) -> InsightDataChecksSample | None:@@ -587,14 +680,6 @@ def _sample_summary(sample: SampledEvents | Skipped) -> InsightDataChecksSamplereturn InsightDataChecksSample(rate=sample.rate, events=sample.events, distinct_ids=sample.distinct_ids)-def _events_checked(checked: CheckedEvents) -> int | None:- if isinstance(checked.counted, CountedEvents):- return checked.counted.events- if isinstance(checked.sample, SampledEvents):- return round(checked.sample.events / checked.sample.rate)- return None--def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents) -> InsightDataChecksCoverage:return InsightDataChecksCoverage(date_from=window.days_in_range[0],@@ -604,7 +689,7 @@ def _coverage(events: ChartEvents, window: CheckWindow, checked: CheckedEvents)checked_date_to=window.days_to_check[-1],days_checked=len(window.days_to_check),filters_test_accounts=events.filters_test_accounts,- events_checked=_events_checked(checked),+ events_checked=events_on_checked_days(checked),sample=_sample_summary(checked.sample),)diff --git a/products/product_analytics/backend/presentation/insight.py b/products/product_analytics/backend/presentation/insight.pyindex 1f9b40557817..54157f560e3f 100644--- a/products/product_analytics/backend/presentation/insight.py+++ b/products/product_analytics/backend/presentation/insight.py@@ -1734,13 +1734,23 @@ class InsightDataChecksRequestSerializer(serializers.Serializer):)+class InsightDataCheckFilterSerializer(serializers.Serializer):+ key = serializers.CharField(help_text="The property the rule reads, such as `$host`.")+ type = serializers.CharField(help_text="The kind of property, such as `event` or `person`.")+ operator = serializers.CharField(help_text="How the rule compares the property, such as `not_regex`.")+ value = serializers.ListField(+ child=serializers.CharField(), help_text="The values the rule compares the property to."+ )++class InsightDataCheckSerializer(serializers.Serializer):key = serializers.ChoiceField(choices=InsightDataCheckKey.choices, help_text="Which check this is.")level = serializers.ChoiceField(choices=InsightDataCheckLevel.choices,help_text=("`passed` when the check found nothing worth acting on, `warning` for a small share of the events, "- "`problem` for a fifth of the events or more, and `skipped` when the check could not run."+ "`problem` for a fifth of the events or more, and `skipped` when the check could not run. The local "+ "development check is never a `problem`, because some apps serve real users from a local host."),)skip_reason = serializers.ChoiceField(@@ -1775,8 +1785,8 @@ class InsightDataCheckSerializer(serializers.Serializer):allow_null=True,help_text=("The change that fixes it. `filter_internal_users`: set `filterTestAccounts` to true on the insight's "- "query. `open_filter_settings`: change the project's internal and test users filter. Null when there is "- "no single change."+ "query. `open_filter_settings`: change the project's internal and test users filter, for example by "+ "adding `filter_to_add`. Null when there is no single change."),)top_events = serializers.ListField(@@ -1784,12 +1794,67 @@ class InsightDataCheckSerializer(serializers.Serializer):)top_hosts = serializers.ListField(child=serializers.CharField(),- help_text="For the local development check, the local hosts found most often, most affected first.",+ help_text=(+ "For the local development check, the local hosts whose events the project's internal and test users "+ "filter keeps, most affected first."+ ),+ )+ distinct_hosts = serializers.IntegerField(+ allow_null=True,+ help_text=(+ "For the local development check, how many local hosts sent the events in `events`. Null for the other "+ "checks and when the check did not run."+ ),)top_bots = serializers.ListField(child=serializers.CharField(),help_text="For the bot traffic check, the bots and crawlers found most often, most affected first.",)+ distinct_bots = serializers.IntegerField(+ allow_null=True,+ help_text=(+ "For the bot traffic check, how many named bots and crawlers sent the sampled events. Null for the other "+ "checks and when the check did not run."+ ),+ )+ removed_by_filter = serializers.IntegerField(+ allow_null=True,+ help_text=(+ "For the internal users check, the part of `events` that the project's internal and test users filter "+ "removes. It is 0 when the insight already applies the filter. Null for the other checks and when the "+ "check did not run."+ ),+ )+ missed_by_filter = serializers.IntegerField(+ allow_null=True,+ help_text=(+ "For the internal users check, the part of `events` from team emails that the project's internal and "+ "test users filter keeps. Null for the other checks and when the check did not run."+ ),+ )+ top_emails = serializers.ListField(+ child=serializers.CharField(),+ help_text=(+ "For the internal users check, the team emails whose events the project's internal and test users filter "+ "keeps, most affected first. Only emails the caller may see as the team's are matched. Empty for a token "+ "without the `person:read` scope."+ ),+ )+ distinct_emails = serializers.IntegerField(+ allow_null=True,+ help_text=(+ "For the internal users check, how many team emails sent events that the project's internal and test "+ "users filter keeps. Null for the other checks and when the check did not run."+ ),+ )+ filter_to_add = InsightDataCheckFilterSerializer(+ allow_null=True,+ help_text=(+ "A rule for the project's internal and test users filter that removes what the check found: the local "+ "hosts or team emails it found, up to 50 values, or bot and AI agent traffic. Null when the check found "+ "none, and for the internal users check for a token without the `person:read` scope."+ ),+ )class InsightDataChecksSampleSerializer(serializers.Serializer):@@ -2582,8 +2647,8 @@ When set, the specified dashboard's filters and date range override will be appl"and lifecycle insights whose series are events. The checks apply the insight's own filters, and read ""the newest days of its date range, up to 7. The bot and duplicate checks read a sample of those days. ""For a token, the internal users check matches the emails of the organization's members only with the "- "`organization_member:read` scope, and its verified domains only with `organization:read`. Nothing is "- "stored. This route runs the query it is sent, so it needs query access as well as insight access. To "+ "`organization_member:read` scope, and its verified domains only with `organization:read`. It names the "+ "emails it finds only with `person:read`. Nothing is stored. This route runs the query it is sent, so it needs query access as well as insight access. To ""check a saved insight by its id, send a GET request to `insights/{id}/data_checks/`."),)@@ -2641,12 +2706,12 @@ When set, the specified dashboard's filters and date range override will be applproject_scoped = bool(get_authenticator_scoped_team_ids(authenticator))def holds(scope: str) -> bool:- if project_scoped:- return Falsereturn scopes is None or "*" in scopes or not scopes_not_covered(scopes, [scope])return TeamEmailAccess(- member_emails=holds("organization_member:read"), verified_domains=holds("organization:read")+ member_emails=not project_scoped and holds("organization_member:read"),+ verified_domains=not project_scoped and holds("organization:read"),+ person_emails=holds("person:read"),)# `Sequence` rather than `list` here: this viewset defines a `list` method that shadows the builtin `list`diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.pyindex 26abcc59a1d4..f983aee56ec1 100644--- a/products/product_analytics/backend/tests/test_insight_data_checks.py+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py@@ -91,27 +91,34 @@ def checked(*,removed_by_filter: int = 0,missed_by_filter: int = 0,- local: int | None = 0,+ counted: bool = True,+ local: int = 0,bots: int = 0,repeats: int = 0,sample_rate: float = 1.0,) -> CheckedEvents:- counted: CountedEvents | Skipped = Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS)- if local is not None:- counted = CountedEvents(+ counted_events: CountedEvents | Skipped = Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS)+ if counted:+ counted_events = CountedEvents(events=round(sampled_events / sample_rate),- local=LocalEvents(count=local, top_hosts=[], top_events=[]),internal=InternalEvents(- removed_by_filter=removed_by_filter, missed_by_filter=missed_by_filter, top_events=[]+ removed_by_filter=removed_by_filter,+ missed_by_filter=missed_by_filter,+ top_events=[],+ distinct_missed_emails=0,+ top_missed_emails=[],+ missed_emails=[],),)return CheckedEvents(- counted=counted,+ counted=counted_events,+ local=LocalEvents(count=local, distinct_hosts=1 if local else 0, top_hosts=[], top_events=[], hosts=[]),sample=SampledEvents(rate=sample_rate,events=sampled_events,distinct_ids=sampled_events,bot_events=bots,+ distinct_bots=0,top_bots=[],top_bot_events=[],repeated_events=repeats,@@ -128,8 +135,8 @@ class TestEvaluateDataChecks(SimpleTestCase):("internal_fifth_is_a_problem", checked(1000, missed_by_filter=200), {INTERNAL: PROBLEM}),("local_tiny_share_still_counts", checked(1_000_000, local=60), {LOCAL: WARNING}),("local_under_50_events", checked(100, local=49), {}),- ("local_fifth_is_a_problem", checked(1000, local=200), {LOCAL: PROBLEM}),- ("too_many_events_to_count", checked(1000, local=None), {INTERNAL: SKIPPED, LOCAL: SKIPPED}),+ ("local_fifth_is_still_a_warning", checked(1000, local=200), {LOCAL: WARNING}),+ ("too_many_events_to_count_skips_only_internal_users", checked(1000, counted=False), {INTERNAL: SKIPPED}),("bots_small_share_is_a_warning", checked(10_000, bots=200), {BOTS: WARNING}),("bots_under_50_sampled_events_but_over_50_in_the_range",@@ -168,7 +175,7 @@ class TestEvaluateDataChecks(SimpleTestCase):@parameterized.expand([- ("local_with_too_many_events_to_count", LOCAL, checked(1000, local=None), "too_many_events"),+ ("internal_users_with_too_many_events_to_count", INTERNAL, checked(1000, counted=False), "too_many_events"),("bots_with_too_few_sampled_events", BOTS, checked(500, sample_rate=0.1), "too_few_sampled_events"),("bots_when_the_insight_filters_on_traffic_type",@@ -212,6 +219,7 @@ class TestEvaluateDataChecks(SimpleTestCase):checks = evaluate_data_checks(CheckedEvents(counted=Skipped(reason=InsightDataCheckSkipReason.TOO_MANY_EVENTS),+ local=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),sample=Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT),))@@ -321,7 +329,7 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):query,team=self.team,user=self.user,- email_access=TeamEmailAccess(member_emails=True, verified_domains=True),+ email_access=TeamEmailAccess(member_emails=True, verified_domains=True, person_emails=True),)def _internal_users(self, result: InsightDataChecks) -> InsightDataCheck:@@ -347,11 +355,14 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):"bot_traffic": ("warning", 60),"duplicate_events": ("warning", 110),}- assert {check.key: (check.top_events, check.top_hosts, check.top_bots) for check in result.checks} == {- "internal_users": ([EVENT], [], []),- "local_development": ([EVENT], [LOCAL_HOST], []),- "bot_traffic": ([EVENT], [], ["Google fetcher", "Googlebot"]),- "duplicate_events": ([EVENT], [], []),+ assert {+ check.key: (check.top_events, check.top_hosts, check.distinct_hosts, check.top_bots)+ for check in result.checks+ } == {+ "internal_users": ([EVENT], [], None, []),+ "local_development": ([EVENT], [LOCAL_HOST], 1, []),+ "bot_traffic": ([EVENT], [], None, ["Google fetcher", "Googlebot"]),+ "duplicate_events": ([EVENT], [], None, []),}@parameterized.expand(@@ -509,7 +520,7 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):@parameterized.expand([- ("events_on_every_day", range(20), "problem", 70),+ ("events_on_every_day", range(20), "warning", 70),("no_events_on_the_newest_days", range(10, 20), "no_events", 0),])@@ -594,13 +605,17 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):@parameterized.expand([- ("insight_without_the_filter", True, {}, (60, "filter_internal_users")),- ("insight_with_the_filter", True, {"filterTestAccounts": True}, (0, None)),- ("project_without_a_filter", False, {}, (60, "open_filter_settings")),+ ("insight_without_the_filter", True, {}, (60, "filter_internal_users", 60, 0)),+ ("insight_with_the_filter", True, {"filterTestAccounts": True}, (0, None, 0, 0)),+ ("project_without_a_filter", False, {}, (60, "open_filter_settings", 0, 60)),])def test_internal_users_check_follows_the_filter_in_place(- self, _name: str, project_has_a_filter: bool, source: dict[str, Any], expected: tuple[int, str | None]+ self,+ _name: str,+ project_has_a_filter: bool,+ source: dict[str, Any],+ expected: tuple[int, str | None, int, int],) -> None:if not project_has_a_filter:self.team.test_account_filters = []@@ -608,7 +623,60 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):internal_users = self._internal_users(self._check(self._query(**source)))- assert (internal_users.events, internal_users.action) == expected+ assert (+ internal_users.events,+ internal_users.action,+ internal_users.removed_by_filter,+ internal_users.missed_by_filter,+ ) == expected++ @parameterized.expand(+ [+ ("persons_are_on_events", "person_id_override_properties_on_events"),+ ("persons_are_not_on_events", "disabled"),+ ("persons_are_joined", "person_id_override_properties_joined"),+ ]+ )+ def test_internal_users_suggested_rule_removes_the_emails_it_names(+ self, _name: str, persons_on_events_mode: str+ ) -> None:+ _create_person(team=self.team, distinct_ids=["qa"], properties={"email": "[email protected]"})+ self._events(60, distinct_id="qa", person_properties={"email": "[email protected]"})+ flush_persons_and_events()+ modifiers = {"personsOnEventsMode": persons_on_events_mode}+ rule = self._internal_users(self._check(self._query(modifiers=modifiers))).filter_to_add+ assert rule is not None+ self.team.test_account_filters = [+ *self.team.test_account_filters,+ {"key": rule.key, "type": rule.type, "operator": rule.operator, "value": rule.value},+ ]+ self.team.save()++ result = self._check(self._query(filterTestAccounts=True, modifiers=modifiers))++ assert self._internal_users(result).missed_by_filter == 0++ @parameterized.expand(+ [+ ("suggested_rule_with_the_insight_filter_off", {}),+ ("suggested_rule_with_the_insight_filter_on", {"filterTestAccounts": True}),+ ]+ )+ def test_local_development_check_counts_only_the_local_events_the_filter_keeps(+ self, _name: str, source: dict[str, Any]+ ) -> None:+ found = next(check for check in self._check(self._query(**source)).checks if check.key == LOCAL)+ assert found.filter_to_add is not None+ rule = found.filter_to_add+ self.team.test_account_filters = [+ *self.team.test_account_filters,+ {"key": rule.key, "type": rule.type, "operator": rule.operator, "value": rule.value},+ ]+ self.team.save()++ local = next(check for check in self._check(self._query(**source)).checks if check.key == LOCAL)++ assert (local.events, local.skip_reason) == (0, None)@parameterized.expand([("without_a_project_filter", None), ("with_a_person_filter", "person"), ("with_a_cohort_filter", "cohort")]@@ -791,36 +859,49 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):@parameterized.expand([- ("member_for_a_session_that_may_see_them", MEMBER_EMAIL, None, True, 80),+ ("member_for_a_session_that_may_see_them", MEMBER_EMAIL, None, True, (80, [MEMBER_EMAIL])),("member_for_a_token_with_the_member_scope",MEMBER_EMAIL,+ [*CHECK_SCOPES, "person:read", "organization_member:read"],+ True,+ (80, [MEMBER_EMAIL]),+ ),+ (+ "member_for_a_token_without_the_member_scope",+ MEMBER_EMAIL,+ [*CHECK_SCOPES, "person:read"],+ True,+ (0, []),+ ),+ ("member_for_a_session_that_may_not_see_them", MEMBER_EMAIL, None, False, (0, [])),+ (+ "member_for_a_token_without_the_person_scope",+ MEMBER_EMAIL,[*CHECK_SCOPES, "organization_member:read"],True,- 80,+ (80, []),),- ("member_for_a_token_without_the_member_scope", MEMBER_EMAIL, CHECK_SCOPES, True, 0),- ("member_for_a_session_that_may_not_see_them", MEMBER_EMAIL, None, False, 0),("verified_domain_for_a_token_with_the_organization_scope",VERIFIED_DOMAIN_EMAIL,- [*CHECK_SCOPES, "organization:read"],+ [*CHECK_SCOPES, "person:read", "organization:read"],True,- 80,+ (80, [VERIFIED_DOMAIN_EMAIL]),),("verified_domain_for_a_token_without_the_organization_scope",VERIFIED_DOMAIN_EMAIL,- [*CHECK_SCOPES, "organization_member:read"],+ [*CHECK_SCOPES, "person:read", "organization_member:read"],True,- 0,+ (0, []),),("member_for_a_token_confined_to_the_project",MEMBER_EMAIL,- [*CHECK_SCOPES, "organization_member:read", "organization:read"],+ [*CHECK_SCOPES, "person:read", "organization_member:read", "organization:read"],True,- 0,+ (0, []),True,),]@@ -831,7 +912,7 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):email: str,scopes: list[str] | None,member_is_visible: bool,- internal_user_events: int,+ expected: tuple[int, list[str]],confined_to_the_project: bool = False,) -> None:self._create_user(MEMBER_EMAIL)@@ -857,7 +938,13 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):assert response.status_code == status.HTTP_200_OK, response.contentinternal_users = next(check for check in response.json()["checks"] if check["key"] == INTERNAL)- assert internal_users["events"] == internal_user_events+ rule = internal_users["filter_to_add"]+ events, named_emails = expected+ assert (internal_users["events"], internal_users["top_emails"], rule["value"] if rule else []) == (+ events,+ named_emails,+ named_emails,+ )def _limit_the_member_list_to_project_members(self) -> None:self.organization.available_product_features = [diff --git a/products/product_analytics/frontend/generated/api.schemas.ts b/products/product_analytics/frontend/generated/api.schemas.tsindex 8a64d278d4ec..9481a71ae319 100644--- a/products/product_analytics/frontend/generated/api.schemas.ts+++ b/products/product_analytics/frontend/generated/api.schemas.ts@@ -9282,6 +9282,17 @@ export const InsightDataCheckActionEnumApi = {OpenFilterSettings: 'open_filter_settings',} as const+export interface InsightDataCheckFilterApi {+ /** The property the rule reads, such as `$host`. */+ key: string+ /** The kind of property, such as `event` or `person`. */+ type: string+ /** How the rule compares the property, such as `not_regex`. */+ operator: string+ /** The values the rule compares the property to. */+ value: string[]+}+export interface InsightDataCheckApi {/** Which check this is.*@@ -9290,7 +9301,7 @@ export interface InsightDataCheckApi {* * `bot_traffic` - Bot and crawler traffic* * `duplicate_events` - Events sent twice */key: InsightDataCheckKeyEnumApi- /** `passed` when the check found nothing worth acting on, `warning` for a small share of the events, `problem` for a fifth of the events or more, and `skipped` when the check could not run.+ /** `passed` when the check found nothing worth acting on, `warning` for a small share of the events, `problem` for a fifth of the events or more, and `skipped` when the check could not run. The local development check is never a `problem`, because some apps serve real users from a local host.** * `passed` - Passed* * `warning` - Warning@@ -9325,17 +9336,46 @@ export interface InsightDataCheckApi {share: number | null/** True when `events` is an estimate scaled up from a sample, rather than a count of every event. */scaled: boolean- /** The change that fixes it. `filter_internal_users`: set `filterTestAccounts` to true on the insight's query. `open_filter_settings`: change the project's internal and test users filter. Null when there is no single change.+ /** The change that fixes it. `filter_internal_users`: set `filterTestAccounts` to true on the insight's query. `open_filter_settings`: change the project's internal and test users filter, for example by adding `filter_to_add`. Null when there is no single change.** * `filter_internal_users` - Turn on the internal and test users filter for the insight* * `open_filter_settings` - Change the project's internal and test users filter */action: InsightDataCheckActionEnumApi | null/** The event names the check found most often, most affected first. */top_events: string[]- /** For the local development check, the local hosts found most often, most affected first. */+ /** For the local development check, the local hosts whose events the project's internal and test users filter keeps, most affected first. */top_hosts: string[]+ /**+ * For the local development check, how many local hosts sent the events in `events`. Null for the other checks and when the check did not run.+ * @nullable+ */+ distinct_hosts: number | null/** For the bot traffic check, the bots and crawlers found most often, most affected first. */top_bots: string[]+ /**+ * For the bot traffic check, how many named bots and crawlers sent the sampled events. Null for the other checks and when the check did not run.+ * @nullable+ */+ distinct_bots: number | null+ /**+ * For the internal users check, the part of `events` that the project's internal and test users filter removes. It is 0 when the insight already applies the filter. Null for the other checks and when the check did not run.+ * @nullable+ */+ removed_by_filter: number | null+ /**+ * For the internal users check, the part of `events` from team emails that the project's internal and test users filter keeps. Null for the other checks and when the check did not run.+ * @nullable+ */+ missed_by_filter: number | null+ /** For the internal users check, the team emails whose events the project's internal and test users filter keeps, most affected first. Only emails the caller may see as the team's are matched. Empty for a token without the `person:read` scope. */+ top_emails: string[]+ /**+ * For the internal users check, how many team emails sent events that the project's internal and test users filter keeps. Null for the other checks and when the check did not run.+ * @nullable+ */+ distinct_emails: number | null+ /** A rule for the project's internal and test users filter that removes what the check found: the local hosts or team emails it found, up to 50 values, or bot and AI agent traffic. Null when the check found none, and for the internal users check for a token without the `person:read` scope. */+ filter_to_add: InsightDataCheckFilterApi | null}export interface InsightDataChecksSampleApi {diff --git a/products/product_analytics/frontend/generated/api.ts b/products/product_analytics/frontend/generated/api.tsindex 159c0bb59847..7f83cdbb36e7 100644--- a/products/product_analytics/frontend/generated/api.ts+++ b/products/product_analytics/frontend/generated/api.ts@@ -849,7 +849,7 @@ export const getInsightsDataChecksCreateUrl = (projectId: string, params?: Insig}/**- * Check the events behind an insight for the common ways they mislead: internal and test users, local development traffic, bots and crawlers, and events sent twice. Works for trends, funnels, stickiness and lifecycle insights whose series are events. The checks apply the insight's own filters, and read the newest days of its date range, up to 7. The bot and duplicate checks read a sample of those days. For a token, the internal users check matches the emails of the organization's members only with the `organization_member:read` scope, and its verified domains only with `organization:read`. Nothing is stored. This route runs the query it is sent, so it needs query access as well as insight access. To check a saved insight by its id, send a GET request to `insights/{id}/data_checks/`.+ * Check the events behind an insight for the common ways they mislead: internal and test users, local development traffic, bots and crawlers, and events sent twice. Works for trends, funnels, stickiness and lifecycle insights whose series are events. The checks apply the insight's own filters, and read the newest days of its date range, up to 7. The bot and duplicate checks read a sample of those days. For a token, the internal users check matches the emails of the organization's members only with the `organization_member:read` scope, and its verified domains only with `organization:read`. It names the emails it finds only with `person:read`. Nothing is stored. This route runs the query it is sent, so it needs query access as well as insight access. To check a saved insight by its id, send a GET request to `insights/{id}/data_checks/`.*/export const insightsDataChecksCreate = async (projectId: string,diff --git a/products/product_analytics/frontend/generated/api.zod.ts b/products/product_analytics/frontend/generated/api.zod.tsindex 9d7444b36951..d3af5a5bcb33 100644--- a/products/product_analytics/frontend/generated/api.zod.ts+++ b/products/product_analytics/frontend/generated/api.zod.ts@@ -287,7 +287,7 @@ export const InsightsCancelCreateBody = /* @__PURE__ */ zod.describe('Deep\/recursive schema (opaque in Zod — use TypeScript types for full shape)')/**- * Check the events behind an insight for the common ways they mislead: internal and test users, local development traffic, bots and crawlers, and events sent twice. Works for trends, funnels, stickiness and lifecycle insights whose series are events. The checks apply the insight's own filters, and read the newest days of its date range, up to 7. The bot and duplicate checks read a sample of those days. For a token, the internal users check matches the emails of the organization's members only with the `organization_member:read` scope, and its verified domains only with `organization:read`. Nothing is stored. This route runs the query it is sent, so it needs query access as well as insight access. To check a saved insight by its id, send a GET request to `insights/{id}/data_checks/`.+ * Check the events behind an insight for the common ways they mislead: internal and test users, local development traffic, bots and crawlers, and events sent twice. Works for trends, funnels, stickiness and lifecycle insights whose series are events. The checks apply the insight's own filters, and read the newest days of its date range, up to 7. The bot and duplicate checks read a sample of those days. For a token, the internal users check matches the emails of the organization's members only with the `organization_member:read` scope, and its verified domains only with `organization:read`. It names the emails it finds only with `person:read`. Nothing is stored. This route runs the query it is sent, so it needs query access as well as insight access. To check a saved insight by its id, send a GET request to `insights/{id}/data_checks/`.*/export const InsightsDataChecksCreateBody = /* @__PURE__ */ zod.record(zod.string(), zod.unknown())diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.tsindex 37b42ce454f9..829aa9ea47b4 100644--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts@@ -13,7 +13,14 @@ const passed: InsightDataCheckApi = {action: null,top_events: [],top_hosts: [],+ distinct_hosts: null,top_bots: [],+ distinct_bots: null,+ removed_by_filter: null,+ missed_by_filter: null,+ top_emails: [],+ distinct_emails: null,+ filter_to_add: null,}describe('dataCheckText', () => {diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex ebbdcd217c16..c4bb995237e4 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -61427,6 +61427,17 @@ export namespace Schemas {OpenFilterSettings: 'open_filter_settings',} as const;+ export interface InsightDataCheckFilter {+ /** The property the rule reads, such as `$host`. */+ key: string;+ /** The kind of property, such as `event` or `person`. */+ type: string;+ /** How the rule compares the property, such as `not_regex`. */+ operator: string;+ /** The values the rule compares the property to. */+ value: string[];+ }+export interface InsightDataCheck {/** Which check this is.*@@ -61435,7 +61446,7 @@ export namespace Schemas {* * `bot_traffic` - Bot and crawler traffic* * `duplicate_events` - Events sent twice */key: InsightDataCheckKeyEnum;- /** `passed` when the check found nothing worth acting on, `warning` for a small share of the events, `problem` for a fifth of the events or more, and `skipped` when the check could not run.+ /** `passed` when the check found nothing worth acting on, `warning` for a small share of the events, `problem` for a fifth of the events or more, and `skipped` when the check could not run. The local development check is never a `problem`, because some apps serve real users from a local host.** * `passed` - Passed* * `warning` - Warning@@ -61470,17 +61481,46 @@ export namespace Schemas {share: number | null;/** True when `events` is an estimate scaled up from a sample, rather than a count of every event. */scaled: boolean;- /** The change that fixes it. `filter_internal_users`: set `filterTestAccounts` to true on the insight's query. `open_filter_settings`: change the project's internal and test users filter. Null when there is no single change.+ /** The change that fixes it. `filter_internal_users`: set `filterTestAccounts` to true on the insight's query. `open_filter_settings`: change the project's internal and test users filter, for example by adding `filter_to_add`. Null when there is no single change.** * `filter_internal_users` - Turn on the internal and test users filter for the insight* * `open_filter_settings` - Change the project's internal and test users filter */action: InsightDataCheckActionEnum | null;/** The event names the check found most often, most affected first. */top_events: string[];- /** For the local development check, the local hosts found most often, most affected first. */+ /** For the local development check, the local hosts whose events the project's internal and test users filter keeps, most affected first. */top_hosts: string[];+ /**+ * For the local development check, how many local hosts sent the events in `events`. Null for the other checks and when the check did not run.+ * @nullable+ */+ distinct_hosts: number | null;/** For the bot traffic check, the bots and crawlers found most often, most affected first. */top_bots: string[];+ /**+ * For the bot traffic check, how many named bots and crawlers sent the sampled events. Null for the other checks and when the check did not run.+ * @nullable+ */+ distinct_bots: number | null;+ /**+ * For the internal users check, the part of `events` that the project's internal and test users filter removes. It is 0 when the insight already applies the filter. Null for the other checks and when the check did not run.+ * @nullable+ */+ removed_by_filter: number | null;+ /**+ * For the internal users check, the part of `events` from team emails that the project's internal and test users filter keeps. Null for the other checks and when the check did not run.+ * @nullable+ */+ missed_by_filter: number | null;+ /** For the internal users check, the team emails whose events the project's internal and test users filter keeps, most affected first. Only emails the caller may see as the team's are matched. Empty for a token without the `person:read` scope. */+ top_emails: string[];+ /**+ * For the internal users check, how many team emails sent events that the project's internal and test users filter keeps. Null for the other checks and when the check did not run.+ * @nullable+ */+ distinct_emails: number | null;+ /** A rule for the project's internal and test users filter that removes what the check found: the local hosts or team emails it found, up to 50 values, or bot and AI agent traffic. Null when the check found none, and for the internal users check for a token without the `person:read` scope. */+ filter_to_add: InsightDataCheckFilter | null;}/**