:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(product-analytics): pass localhost check when nearly every event is local (#125388)
Sam Pennington87e5b3a665ec
diff --git a/products/product_analytics/backend/insight_data_check_results.py b/products/product_analytics/backend/insight_data_check_results.pyindex 3fcd3cbdfc8f..bd4e0217520d 100644--- a/products/product_analytics/backend/insight_data_check_results.py+++ b/products/product_analytics/backend/insight_data_check_results.py@@ -19,6 +19,8 @@ MIN_DUPLICATE_EVENTS = 100# Higher than the other share floors, because events that differ only in custom properties count as repeats.MIN_DUPLICATE_SHARE = 0.05PROBLEM_SHARE = 0.2+# Kept in sync with LOCAL_APP_SHARE in dataCheckText.ts.+LOCAL_APP_SHARE = 0.9# An event with no user agent has the type "Automation", and server-side SDKs send those, so it is not a crawler.CRAWLER_TRAFFIC_TYPES = ("Bot", "AI Agent")@@ -173,7 +175,11 @@ def _local_development_check(checked: CheckedEvents) -> InsightDataCheck:events = events_on_checked_days(checked)share = _share(local.count, events) if events is not None else None# No minimum share: a local session also creates test persons and flag exposures, whatever its share.- is_finding = local.count >= MIN_FINDING_EVENTS+ # When nearly every event is local, the events come from the app itself, such as a desktop app, and a filter+ # that removes them would empty the chart.+ # ponytail: a share heuristic. A project with only local development events also passes. Upgrade to a dismiss+ # per project if people need it.+ is_finding = local.count >= MIN_FINDING_EVENTS and (share is None or share < LOCAL_APP_SHARE)return InsightDataCheck(key=key,# Never a problem: a project can serve real users from a local host, such as an app in a native wrapper,diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.pyindex 9f50fa6504ad..936bef5313ea 100644--- a/products/product_analytics/backend/tests/test_insight_data_checks.py+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py@@ -150,6 +150,8 @@ class TestEvaluateDataChecks(SimpleTestCase):("local_tiny_share_still_counts", checked(1_000_000, local=60), {LOCAL: WARNING}),("local_under_50_events", checked(100, local=49), {}),("local_fifth_is_still_a_warning", checked(1000, local=200), {LOCAL: WARNING}),+ ("local_under_nine_tenths_is_still_a_warning", checked(1000, local=899), {LOCAL: WARNING}),+ ("local_nearly_every_event_is_the_app_itself", checked(1000, local=900), {}),("too_many_events_to_count_skips_only_internal_users", checked(1000, counted=False), {INTERNAL: SKIPPED}),("bots_small_share_is_a_warning", checked(10_000, bots=200), {BOTS: WARNING}),(@@ -562,7 +564,8 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):self, _name: str, days_with_events: range, status: str, events_checked: int) -> None:for days_ago in days_with_events:- self._events(10, event=WIDELY_SENT_EVENT, days_ago=days_ago, properties={"$host": LOCAL_HOST})+ self._events(8, event=WIDELY_SENT_EVENT, days_ago=days_ago, properties={"$host": LOCAL_HOST})+ self._events(2, event=WIDELY_SENT_EVENT, days_ago=days_ago)flush_persons_and_events()query = self._query([{"kind": "EventsNode", "event": WIDELY_SENT_EVENT}],diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.tsindex 6500a21d5210..20524e969da2 100644--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts@@ -42,6 +42,13 @@ describe('dataCheckText', () => {false,'Almost none (about 1,234 events)',],+ [+ 'a passed localhost check where nearly every event is local',+ { key: 'local_development', method: 'counted', events: 950, share: 0.95 },+ false,+ false,+ '95% of these events, so they come from your app',+ ],['internal users on an insight that filters them out',{ key: 'internal_users', method: 'counted' },diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.tsindex 028385dc9888..06f6c353582a 100644--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts@@ -22,6 +22,8 @@ export interface DataCheckText {const TRAFFIC_TYPE_PROPERTY = '$virt_traffic_type'// The traffic types the bot check counts, as the backend's CRAWLER_TRAFFIC_TYPES lists them.const CRAWLER_TRAFFIC_TYPES = ['Bot', 'AI Agent']+// The share from which the localhost check passes, as the backend's LOCAL_APP_SHARE sets it.+const LOCAL_APP_SHARE = 0.9export function removesBotTraffic(rules: readonly { key?: string; type?: string; operator?: unknown; value?: unknown }[]@@ -69,6 +71,11 @@ function percentOf(share: number | null): string | null {return share !== null && share >= 0.01 ? percentage(share, 0) : null}+function amountOf(check: InsightDataCheckApi): string {+ const percent = percentOf(check.share)+ return percent ? `${check.scaled ? 'About ' : ''}${percent}` : capitalizeFirstLetter(eventCount(check))+}+function shareOf(check: InsightDataCheckApi, part: number): string | null {return percentOf(check.share !== null && check.events ? (check.share * part) / check.events : null)}@@ -234,6 +241,20 @@ function skippedResult(check: InsightDataCheckApi): string {return reason ? `Not checked, because ${reason}` : 'Not checked'}+function passedResult(check: InsightDataCheckApi, filtersTestAccounts: boolean): string {+ if (check.key === 'internal_users' && filtersTestAccounts) {+ if (!check.missed_by_filter) {+ return 'Filtered out'+ }+ const missed = missedUserCount(check)+ return `Filtered out, except ${missed ? pluralize(missed, 'user') : 'some users'}`+ }+ if (check.key === 'local_development' && (check.share ?? 0) >= LOCAL_APP_SHARE) {+ return `${amountOf(check)} of these events, so they come from your app`+ }+ return check.events ? `Almost none (${eventCount(check)} events)` : 'None found'+}+// The server skips the bot check whenever the insight's filters read traffic type, and cannot tell when the rule// comes from the project's filter.export function botsFilteredOut(@@ -257,16 +278,7 @@ export function dataCheckText(return { name, result: skippedResult(check) }}if (check.level === 'passed') {- if (check.key === 'internal_users' && filtersTestAccounts) {- if (!check.missed_by_filter) {- return { name, result: 'Filtered out' }- }- const missed = missedUserCount(check)- return { name, result: `Filtered out, except ${missed ? pluralize(missed, 'user') : 'some users'}` }- }- return { name, result: check.events ? `Almost none (${eventCount(check)} events)` : 'None found' }+ return { name, result: passedResult(check, filtersTestAccounts) }}- const percent = percentOf(check.share)- const amount = percent ? `${check.scaled ? 'About ' : ''}${percent}` : capitalizeFirstLetter(eventCount(check))- return { name, result: `${amount} of these events` }+ return { name, result: `${amountOf(check)} of these events` }}