Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(product-analytics): pass localhost check when nearly every event is local (#125388)

Sam Pennington87e5b3a665ec
diff --git a/products/product_analytics/backend/insight_data_check_results.py b/products/product_analytics/backend/insight_data_check_results.py
index 3fcd3cbdfc8f..bd4e0217520d 100644
--- a/products/product_analytics/backend/insight_data_check_results.py
+++ b/products/product_analytics/backend/insight_data_check_results.py
@@ -19,6 +19,8 @@ MIN_DUPLICATE_EVENTS = 100
# Higher than the other share floors, because events that differ only in custom properties count as repeats.
MIN_DUPLICATE_SHARE = 0.05
PROBLEM_SHARE = 0.2
+# Kept in sync with LOCAL_APP_SHARE in dataCheckText.ts.
+LOCAL_APP_SHARE = 0.9
# An event with no user agent has the type "Automation", and server-side SDKs send those, so it is not a crawler.
CRAWLER_TRAFFIC_TYPES = ("Bot", "AI Agent")
@@ -173,7 +175,11 @@ def _local_development_check(checked: CheckedEvents) -> InsightDataCheck:
events = events_on_checked_days(checked)
share = _share(local.count, events) if events is not None else None
# No minimum share: a local session also creates test persons and flag exposures, whatever its share.
- is_finding = local.count >= MIN_FINDING_EVENTS
+ # When nearly every event is local, the events come from the app itself, such as a desktop app, and a filter
+ # that removes them would empty the chart.
+ # ponytail: a share heuristic. A project with only local development events also passes. Upgrade to a dismiss
+ # per project if people need it.
+ is_finding = local.count >= MIN_FINDING_EVENTS and (share is None or share < LOCAL_APP_SHARE)
return InsightDataCheck(
key=key,
# Never a problem: a project can serve real users from a local host, such as an app in a native wrapper,
diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.py
index 9f50fa6504ad..936bef5313ea 100644
--- a/products/product_analytics/backend/tests/test_insight_data_checks.py
+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py
@@ -150,6 +150,8 @@ class TestEvaluateDataChecks(SimpleTestCase):
("local_tiny_share_still_counts", checked(1_000_000, local=60), {LOCAL: WARNING}),
("local_under_50_events", checked(100, local=49), {}),
("local_fifth_is_still_a_warning", checked(1000, local=200), {LOCAL: WARNING}),
+ ("local_under_nine_tenths_is_still_a_warning", checked(1000, local=899), {LOCAL: WARNING}),
+ ("local_nearly_every_event_is_the_app_itself", checked(1000, local=900), {}),
("too_many_events_to_count_skips_only_internal_users", checked(1000, counted=False), {INTERNAL: SKIPPED}),
("bots_small_share_is_a_warning", checked(10_000, bots=200), {BOTS: WARNING}),
(
@@ -562,7 +564,8 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):
self, _name: str, days_with_events: range, status: str, events_checked: int
) -> None:
for days_ago in days_with_events:
- self._events(10, event=WIDELY_SENT_EVENT, days_ago=days_ago, properties={"$host": LOCAL_HOST})
+ self._events(8, event=WIDELY_SENT_EVENT, days_ago=days_ago, properties={"$host": LOCAL_HOST})
+ self._events(2, event=WIDELY_SENT_EVENT, days_ago=days_ago)
flush_persons_and_events()
query = self._query(
[{"kind": "EventsNode", "event": WIDELY_SENT_EVENT}],
diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts
index 6500a21d5210..20524e969da2 100644
--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts
@@ -42,6 +42,13 @@ describe('dataCheckText', () => {
false,
'Almost none (about 1,234 events)',
],
+ [
+ 'a passed localhost check where nearly every event is local',
+ { key: 'local_development', method: 'counted', events: 950, share: 0.95 },
+ false,
+ false,
+ '95% of these events, so they come from your app',
+ ],
[
'internal users on an insight that filters them out',
{ key: 'internal_users', method: 'counted' },
diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
index 028385dc9888..06f6c353582a 100644
--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
@@ -22,6 +22,8 @@ export interface DataCheckText {
const TRAFFIC_TYPE_PROPERTY = '$virt_traffic_type'
// The traffic types the bot check counts, as the backend's CRAWLER_TRAFFIC_TYPES lists them.
const CRAWLER_TRAFFIC_TYPES = ['Bot', 'AI Agent']
+// The share from which the localhost check passes, as the backend's LOCAL_APP_SHARE sets it.
+const LOCAL_APP_SHARE = 0.9
export function removesBotTraffic(
rules: readonly { key?: string; type?: string; operator?: unknown; value?: unknown }[]
@@ -69,6 +71,11 @@ function percentOf(share: number | null): string | null {
return share !== null && share >= 0.01 ? percentage(share, 0) : null
}
+function amountOf(check: InsightDataCheckApi): string {
+ const percent = percentOf(check.share)
+ return percent ? `${check.scaled ? 'About ' : ''}${percent}` : capitalizeFirstLetter(eventCount(check))
+}
+
function shareOf(check: InsightDataCheckApi, part: number): string | null {
return percentOf(check.share !== null && check.events ? (check.share * part) / check.events : null)
}
@@ -234,6 +241,20 @@ function skippedResult(check: InsightDataCheckApi): string {
return reason ? `Not checked, because ${reason}` : 'Not checked'
}
+function passedResult(check: InsightDataCheckApi, filtersTestAccounts: boolean): string {
+ if (check.key === 'internal_users' && filtersTestAccounts) {
+ if (!check.missed_by_filter) {
+ return 'Filtered out'
+ }
+ const missed = missedUserCount(check)
+ return `Filtered out, except ${missed ? pluralize(missed, 'user') : 'some users'}`
+ }
+ if (check.key === 'local_development' && (check.share ?? 0) >= LOCAL_APP_SHARE) {
+ return `${amountOf(check)} of these events, so they come from your app`
+ }
+ return check.events ? `Almost none (${eventCount(check)} events)` : 'None found'
+}
+
// The server skips the bot check whenever the insight's filters read traffic type, and cannot tell when the rule
// comes from the project's filter.
export function botsFilteredOut(
@@ -257,16 +278,7 @@ export function dataCheckText(
return { name, result: skippedResult(check) }
}
if (check.level === 'passed') {
- if (check.key === 'internal_users' && filtersTestAccounts) {
- if (!check.missed_by_filter) {
- return { name, result: 'Filtered out' }
- }
- const missed = missedUserCount(check)
- return { name, result: `Filtered out, except ${missed ? pluralize(missed, 'user') : 'some users'}` }
- }
- return { name, result: check.events ? `Almost none (${eventCount(check)} events)` : 'None found' }
+ return { name, result: passedResult(check, filtersTestAccounts) }
}
- const percent = percentOf(check.share)
- const amount = percent ? `${check.scaled ? 'About ' : ''}${percent}` : capitalizeFirstLetter(eventCount(check))
- return { name, result: `${amount} of these events` }
+ return { name, result: `${amountOf(check)} of these events` }
}