Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(insights): data check read limits, member fixes and indicator (#120756)

Sam Penningtonc9c6a1276f55
diff --git a/products/product_analytics/backend/insight_data_checks.py b/products/product_analytics/backend/insight_data_checks.py
index 4531a4201139..3435b1c99588 100644
--- a/products/product_analytics/backend/insight_data_checks.py
+++ b/products/product_analytics/backend/insight_data_checks.py
@@ -76,10 +76,10 @@ SIZING_SAMPLE_ONE_IN = 1_000
# A check that takes longer than this is skipped, so it never holds up the insight it sits next to.
MAX_EXECUTION_SECONDS = 5
# Backstops for a size estimate that is far too low, such as for an event sent under a few distinct IDs.
-# ClickHouse applies them to each shard separately.
-MAX_BYTES_READ_PER_SHARD = 1_000_000_000
+# ClickHouse applies them to the total across all shards, so this one sits above what a check reads at EVENT_BUDGET.
+MAX_BYTES_READ = 10_000_000_000
# The sample reads wide columns and sorts them for the repeat window, so it gets a tighter limit.
-MAX_SAMPLE_BYTES_READ_PER_SHARD = 250_000_000
+MAX_SAMPLE_BYTES_READ = 1_000_000_000
# A person cannot repeat an action this fast, but an app that sends an event twice does.
REPEAT_WINDOW_MILLISECONDS = 100
@@ -208,7 +208,7 @@ class _EventReader:
database: Database
def rows_within_limits(
- self, query: ast.SelectQuery, *, query_type: str, max_bytes_read_per_shard: int
+ self, query: ast.SelectQuery, *, query_type: str, max_bytes_read: int
) -> list[list[Any]] | Skipped:
try:
response = execute_hogql_query(
@@ -218,7 +218,7 @@ class _EventReader:
query_type=query_type,
modifiers=self.modifiers,
context=HogQLContext(team_id=self.team.pk, user=self.user, database=self.database),
- settings=_query_settings(max_bytes_read_per_shard),
+ settings=_query_settings(max_bytes_read),
)
except CHQueryErrorTooManyBytes:
return Skipped(reason=InsightDataCheckSkipReason.READ_LIMIT)
@@ -229,12 +229,12 @@ class _EventReader:
return [list(row) for row in response.results or []]
-def _query_settings(max_bytes_read_per_shard: int) -> HogQLGlobalSettings:
+def _query_settings(max_bytes_read: int) -> HogQLGlobalSettings:
# Without "throw", a query that reaches a limit can return the events it counted so far as the answer.
return HogQLGlobalSettings(
max_execution_time=MAX_EXECUTION_SECONDS,
timeout_overflow_mode="throw",
- max_bytes_to_read=max_bytes_read_per_shard,
+ max_bytes_to_read=max_bytes_read,
read_overflow_mode="throw",
)
@@ -326,9 +326,7 @@ def _count_events_per_day(
where=ast.And(exprs=[events.sort_key_where, _between_days(events, days[0], days[-1], timezone)]),
sample=_sample_of(1, SIZING_SAMPLE_ONE_IN),
)
- rows = reader.rows_within_limits(
- query, query_type="insight_data_checks_sizing", max_bytes_read_per_shard=MAX_BYTES_READ_PER_SHARD
- )
+ rows = reader.rows_within_limits(query, query_type="insight_data_checks_sizing", max_bytes_read=MAX_BYTES_READ)
if isinstance(rows, Skipped):
return rows
@@ -524,9 +522,7 @@ def _count_every_event(
)
if match.person_fields is not None:
cast(ast.JoinExpr, query.select_from).next_join = _current_person_join(events, where, match.person_fields)
- rows = reader.rows_within_limits(
- query, query_type="insight_data_checks_counted", max_bytes_read_per_shard=MAX_BYTES_READ_PER_SHARD
- )
+ rows = reader.rows_within_limits(query, query_type="insight_data_checks_counted", max_bytes_read=MAX_BYTES_READ)
if isinstance(rows, Skipped):
return rows
@@ -564,9 +560,7 @@ def _count_local_events_the_filter_keeps(
if person_fields is not None:
local_events = ast.And(exprs=[clone_expr(where), clone_expr(is_local_host)])
cast(ast.JoinExpr, query.select_from).next_join = _current_person_join(events, local_events, person_fields)
- rows = reader.rows_within_limits(
- query, query_type="insight_data_checks_local", max_bytes_read_per_shard=MAX_BYTES_READ_PER_SHARD
- )
+ rows = reader.rows_within_limits(query, query_type="insight_data_checks_local", max_bytes_read=MAX_BYTES_READ)
if isinstance(rows, Skipped):
return rows
@@ -630,7 +624,7 @@ def _sample_events(
rows = reader.rows_within_limits(
cast(ast.SelectQuery, query),
query_type="insight_data_checks_sample",
- max_bytes_read_per_shard=MAX_SAMPLE_BYTES_READ_PER_SHARD,
+ max_bytes_read=MAX_SAMPLE_BYTES_READ,
)
if isinstance(rows, Skipped):
return rows
diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.py
index f983aee56ec1..a89ae7182b2c 100644
--- a/products/product_analytics/backend/tests/test_insight_data_checks.py
+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py
@@ -755,9 +755,9 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):
]
)
def test_reports_why_nothing_was_checked(
- self, _name: str, series: list[dict[str, Any]], max_bytes_read_per_shard: int, expected: str
+ self, _name: str, series: list[dict[str, Any]], max_bytes_read: int, expected: str
) -> None:
- with patch(f"{CHECKS_MODULE}.MAX_BYTES_READ_PER_SHARD", max_bytes_read_per_shard):
+ with patch(f"{CHECKS_MODULE}.MAX_BYTES_READ", max_bytes_read):
result = self._check(self._query(series))
assert result.status == expected
diff --git a/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx b/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx
new file mode 100644
index 000000000000..7e8dded9d655
--- /dev/null
+++ b/products/product_analytics/frontend/insights/dataChecks/DataChecksTag.tsx
@@ -0,0 +1,51 @@
+import { IconCheck, IconCircleDashed, IconWarning } from '@posthog/icons'
+import { LemonTag, LemonTagType } from '@posthog/lemon-ui'
+
+import { Spinner } from 'lib/lemon-ui/Spinner'
+import { cn } from 'lib/utils/css-classes'
+
+import type { InsightDataChecksStatus } from './insightDataChecksLogic'
+
+const TYPES: Partial<Record<InsightDataChecksStatus, LemonTagType>> = {
+ passed: 'success',
+ warning: 'warning',
+ problem: 'danger',
+}
+
+export function DataChecksTag({
+ status,
+ label,
+ labelOpen,
+}: {
+ status: InsightDataChecksStatus
+ label: string
+ labelOpen: boolean
+}): JSX.Element {
+ return (
+ <LemonTag type={TYPES[status] ?? 'muted'} className="gap-0">
+ {status === 'passed' ? (
+ <IconCheck key={status} />
+ ) : status === 'checking' ? (
+ <Spinner key={status} textColored />
+ ) : status === 'no_events' || status === 'inconclusive' ? (
+ <IconCircleDashed key={status} />
+ ) : (
+ <IconWarning key={status} />
+ )}
+ <span
+ className={cn(
+ 'overflow-hidden whitespace-nowrap',
+ 'transition-[max-width,opacity] duration-700 ease-out motion-reduce:duration-0',
+ labelOpen ? 'max-w-40 opacity-100' : 'max-w-0 opacity-0',
+ 'group-hover/checks:max-w-40 group-hover/checks:opacity-100',
+ 'group-focus-visible/checks:max-w-40 group-focus-visible/checks:opacity-100',
+ 'group-data-popup-open/checks:max-w-40 group-data-popup-open/checks:opacity-100'
+ )}
+ >
+ <span key={status} className="ps-1 transition-opacity duration-500 starting:opacity-0">
+ {label}
+ </span>
+ </span>
+ </LemonTag>
+ )
+}
diff --git a/products/product_analytics/frontend/insights/dataChecks/ExcludeFromFilterButton.tsx b/products/product_analytics/frontend/insights/dataChecks/ExcludeFromFilterButton.tsx
index 85e205bdabb6..2359d9a58e91 100644
--- a/products/product_analytics/frontend/insights/dataChecks/ExcludeFromFilterButton.tsx
+++ b/products/product_analytics/frontend/insights/dataChecks/ExcludeFromFilterButton.tsx
@@ -12,9 +12,7 @@ import {
Button,
} from '@posthog/quill'
-import { RestrictionScope, useRestrictedArea } from 'lib/components/RestrictedArea'
import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
-import { TeamMembershipLevel } from 'lib/constants'
import { LinkPrimitive } from 'lib/lemon-ui/Link'
import { urls } from 'scenes/urls'
@@ -28,19 +26,15 @@ import { insightDataChecksLogic } from './insightDataChecksLogic'
export function ExcludeFromFilterButton({
insightProps,
check,
- label,
+ canEditFilter,
}: {
insightProps: InsightLogicProps
check: InsightDataCheckApi
- label: string
+ canEditFilter: boolean
}): JSX.Element {
const logic = insightDataChecksLogic(insightProps)
const { currentTeamLoading } = useValues(logic)
const { addFilterRule } = useActions(logic)
- const restrictedReason = useRestrictedArea({
- scope: RestrictionScope.Project,
- minimumAccessLevel: TeamMembershipLevel.Admin,
- })
const rule = check.filter_to_add
if (!rule) {
@@ -74,12 +68,16 @@ export function ExcludeFromFilterButton({
variant="outline"
size="sm"
loading={currentTeamLoading}
- disabledReason={restrictedReason}
+ disabledReason={
+ canEditFilter
+ ? undefined
+ : 'Only project admins can change the internal and test users filter.'
+ }
data-attr="insight-data-check-add-filter-rule"
/>
}
>
- {label}
+ Add to project filter
</AlertDialogTrigger>
<AlertDialogContent>
<AlertDialogHeader>
diff --git a/products/product_analytics/frontend/insights/dataChecks/ExcludeFromInsightButton.tsx b/products/product_analytics/frontend/insights/dataChecks/ExcludeFromInsightButton.tsx
new file mode 100644
index 000000000000..fe3bd658a42f
--- /dev/null
+++ b/products/product_analytics/frontend/insights/dataChecks/ExcludeFromInsightButton.tsx
@@ -0,0 +1,44 @@
+import { useActions, useValues } from 'kea'
+
+import { Button } from '@posthog/quill'
+
+import type { AnyPropertyFilter, InsightLogicProps } from '~/types'
+
+import type { InsightDataCheckApi } from 'products/product_analytics/frontend/generated/api.schemas'
+
+import { insightDataChecksLogic } from './insightDataChecksLogic'
+import { withPropertyFilter } from './withPropertyFilter'
+
+export function ExcludeFromInsightButton({
+ insightProps,
+ check,
+}: {
+ insightProps: InsightLogicProps
+ check: InsightDataCheckApi
+}): JSX.Element | null {
+ const logic = insightDataChecksLogic(insightProps)
+ const { querySource, status } = useValues(logic)
+ const { excludeFromInsight } = useActions(logic)
+
+ if (!check.filter_to_add) {
+ return null
+ }
+ const fits = withPropertyFilter(querySource?.properties, check.filter_to_add as AnyPropertyFilter) !== null
+
+ return (
+ <Button
+ variant="outline"
+ size="sm"
+ loading={status === 'checking'}
+ disabledReason={
+ fits
+ ? undefined
+ : "This insight's filters use OR groups. Add the rule in the insight's filters instead."
+ }
+ onClick={() => excludeFromInsight(check)}
+ data-attr="insight-data-check-exclude-from-insight"
+ >
+ Exclude from this insight
+ </Button>
+ )
+}
diff --git a/products/product_analytics/frontend/insights/dataChecks/FilterCheckDetails.tsx b/products/product_analytics/frontend/insights/dataChecks/FilterCheckDetails.tsx
index 11f35339be92..3729ea9898ac 100644
--- a/products/product_analytics/frontend/insights/dataChecks/FilterCheckDetails.tsx
+++ b/products/product_analytics/frontend/insights/dataChecks/FilterCheckDetails.tsx
@@ -2,7 +2,9 @@ import { useActions, useValues } from 'kea'
import { Text } from '@posthog/quill'
+import { RestrictionScope, useRestrictedArea } from 'lib/components/RestrictedArea'
import { TestAccountFilterSwitch } from 'lib/components/TestAccountFiltersSwitch'
+import { TeamMembershipLevel } from 'lib/constants'
import type { InsightLogicProps } from '~/types'
@@ -10,6 +12,7 @@ import type { InsightDataCheckApi } from 'products/product_analytics/frontend/ge
import { filterCheckParts } from './dataCheckText'
import { ExcludeFromFilterButton } from './ExcludeFromFilterButton'
+import { ExcludeFromInsightButton } from './ExcludeFromInsightButton'
import { FixWithPostHogAIButton } from './FixWithPostHogAIButton'
import { insightDataChecksLogic } from './insightDataChecksLogic'
import { NotCaughtByFilter } from './NotCaughtByFilter'
@@ -25,6 +28,10 @@ export function FilterCheckDetails({
const { filtersTestAccounts, filterRemovesBots, queryFiltersTestAccounts } = useValues(logic)
const { setFiltersTestAccounts } = useActions(logic)
const { caught, missed } = filterCheckParts(check, filtersTestAccounts, filterRemovesBots)
+ const canEditFilter = !useRestrictedArea({
+ scope: RestrictionScope.Project,
+ minimumAccessLevel: TeamMembershipLevel.Admin,
+ })
return (
<div className="flex w-full flex-col gap-3">
@@ -56,9 +63,14 @@ export function FilterCheckDetails({
<ExcludeFromFilterButton
insightProps={insightProps}
check={check}
- label={missed.excludeLabel}
+ canEditFilter={canEditFilter}
/>
- <FixWithPostHogAIButton insightProps={insightProps} check={check} />
+ {/* PostHog AI would change the project's filter, which only admins may change here. */}
+ {canEditFilter ? (
+ <FixWithPostHogAIButton insightProps={insightProps} check={check} />
+ ) : (
+ <ExcludeFromInsightButton insightProps={insightProps} check={check} />
+ )}
</div>
</div>
)}
diff --git a/products/product_analytics/frontend/insights/dataChecks/InsightDataCheckRow.tsx b/products/product_analytics/frontend/insights/dataChecks/InsightDataCheckRow.tsx
index 63892357b85e..89f9d8659608 100644
--- a/products/product_analytics/frontend/insights/dataChecks/InsightDataCheckRow.tsx
+++ b/products/product_analytics/frontend/insights/dataChecks/InsightDataCheckRow.tsx
@@ -90,9 +90,16 @@ export function InsightDataCheckRow({
}
return (
- <AccordionItem value={check.key} data-attr="insight-data-check" data-check={check.key} data-level={check.level}>
+ <AccordionItem
+ value={check.key}
+ data-attr="insight-data-check"
+ data-check={check.key}
+ data-level={check.level}
+ // The trigger sits in an h3, and the app's base styles give every h3 a bottom margin.
+ className="[&>h3]:m-0"
+ >
<AccordionTrigger className="px-2 py-1 font-normal">{header}</AccordionTrigger>
- <AccordionContent className="flex flex-col items-start gap-2 ps-6">
+ <AccordionContent className="flex flex-col items-start gap-2 ps-6 pt-2">
{foundRepeats && <DuplicateEventsDetails insightProps={insightProps} check={check} />}
{dependsOnFilter && <FilterCheckDetails insightProps={insightProps} check={check} />}
</AccordionContent>
diff --git a/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx b/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx
index a50db1a87817..f98b8dc94080 100644
--- a/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx
+++ b/products/product_analytics/frontend/insights/dataChecks/InsightDataChecks.tsx
@@ -1,22 +1,14 @@
import { useValues } from 'kea'
-import { IconCheck, IconWarning } from '@posthog/icons'
-import { Dot, Spinner, cn } from '@posthog/quill'
-
import { pluralize } from 'lib/utils/strings'
import type { InsightLogicProps } from '~/types'
import { DataChecksPopover } from './DataChecksPopover'
+import { DataChecksTag } from './DataChecksTag'
import { insightDataChecksLogic } from './insightDataChecksLogic'
import type { InsightDataChecksStatus } from './insightDataChecksLogic'
-const COLORS: Partial<Record<InsightDataChecksStatus, string>> = {
- passed: 'border-(--success-foreground) bg-(--success) text-(--success-foreground)',
- warning: 'border-(--warning-foreground) bg-(--warning) text-(--warning-foreground)',
- problem: 'border-(--destructive-foreground) bg-(--destructive) text-(--destructive-foreground)',
-}
-
function statusLabel(status: InsightDataChecksStatus, findingCount: number): string {
switch (status) {
case 'checking':
@@ -43,52 +35,20 @@ export function InsightDataChecks({ insightProps }: { insightProps: InsightLogic
return null
}
+ const label = statusLabel(status, findings.length)
return (
<DataChecksPopover
insightProps={insightProps}
trigger={
<button
type="button"
- data-quill
- className={cn(
- 'group/checks inline-flex cursor-pointer items-center rounded-full border-2 p-0',
- 'transition-colors duration-700 motion-reduce:duration-0',
- 'focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-(--ring)',
- COLORS[status] ?? 'border-(--border) bg-(--muted) text-(--muted-foreground)'
- )}
+ className="group/checks inline-flex cursor-pointer rounded-(--radius) border-0 bg-transparent p-0"
data-attr="insight-data-checks"
data-status={status}
/>
}
>
- <span
- key={status}
- className="inline-flex size-5 shrink-0 items-center justify-center transition-opacity duration-500 starting:opacity-0"
- >
- {status === 'passed' ? (
- <IconCheck className="size-4" />
- ) : status === 'checking' ? (
- <Spinner className="size-3.5" />
- ) : status === 'no_events' || status === 'inconclusive' ? (
- <Dot variant="default" />
- ) : (
- <IconWarning className="size-4" />
- )}
- </span>
- <span
- className={cn(
- 'overflow-hidden text-xs font-medium whitespace-nowrap',
- 'transition-[max-width,opacity] duration-700 ease-out motion-reduce:duration-0',
- labelOpen ? 'max-w-40 opacity-100' : 'max-w-0 opacity-0',
- 'group-hover/checks:max-w-40 group-hover/checks:opacity-100',
- 'group-focus-visible/checks:max-w-40 group-focus-visible/checks:opacity-100',
- 'group-data-popup-open/checks:max-w-40 group-data-popup-open/checks:opacity-100'
- )}
- >
- <span key={status} className="pe-2.5 transition-opacity duration-500 starting:opacity-0">
- {statusLabel(status, findings.length)}
- </span>
- </span>
+ <DataChecksTag status={status} label={label} labelOpen={labelOpen} />
</DataChecksPopover>
)
}
diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts
index c524f5f0173b..2db2c76afffd 100644
--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.test.ts
@@ -85,7 +85,6 @@ describe('dataCheckText', () => {
noun: 'user',
fallback: "Your team's emails",
- excludeLabel: 'Exclude these users',
},
},
],
@@ -122,7 +121,6 @@ describe('dataCheckText', () => {
noun: 'bot',
examples: ['Googlebot', 'Ahrefs'],
fallback: 'Bot events',
- excludeLabel: 'Exclude these bots',
},
},
],
diff --git a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
index 01345690c110..5b36e34208d1 100644
--- a/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/dataCheckText.ts
@@ -75,7 +75,7 @@ function shareOf(check: InsightDataCheckApi, part: number): string | null {
export interface FilterCheckParts {
/** Set when the project's filter catches these events but the insight leaves the filter off. */
caught: string | null
- missed: { count: number; noun: string; examples: string[]; fallback: string; excludeLabel: string } | null
+ missed: { count: number; noun: string; examples: string[]; fallback: string } | null
}
const NOTHING: FilterCheckParts = { caught: null, missed: null }
@@ -99,7 +99,6 @@ function internalUsersParts(check: InsightDataCheckApi, filtersTestAccounts: boo
noun: 'user',
examples: check.top_emails,
fallback: "Your team's emails",
- excludeLabel: 'Exclude these users',
}
: null,
}
@@ -116,7 +115,6 @@ function localHostParts(check: InsightDataCheckApi): FilterCheckParts {
noun: 'host',
examples: check.top_hosts,
fallback: 'Localhost events',
- excludeLabel: 'Exclude these hosts',
},
}
}
@@ -139,7 +137,6 @@ function botTrafficParts(
noun: 'bot',
examples: check.top_bots,
fallback: 'Bot events',
- excludeLabel: 'Exclude these bots',
},
}
}
diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts
index 3290e9d28687..682fa8dd5a40 100644
--- a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts
@@ -211,6 +211,19 @@ describe('insightDataChecksLogic', () => {
expect(logic.values.status).toBe(expected)
})
+ it.each([
+ ['problem', true],
+ ['warning', true],
+ ['passed', false],
+ ])('keeps the label open after a %s result only when there is a finding', async (result, open) => {
+ dataChecks.mockImplementation(answer([200, checksOf(result)]))
+ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()
+
+ logic.actions.closeLabel()
+
+ expect(logic.values.labelOpen).toBe(open)
+ })
+
it('checks again when the chart type changes and the chart reuses its result', async () => {
dataChecks.mockImplementation(answer([200, checksOf('passed')]))
await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()
@@ -299,6 +312,21 @@ describe('insightDataChecksLogic', () => {
const addTheRule = (): void => {
logic.actions.addFilterRule({ key: 'internal_users', filter_to_add: rule } as InsightDataCheckApi)
}
+ it("adds the check's rule to this insight's filters once, then checks the filtered chart", async () => {
+ dataChecks.mockImplementation(answer([200, checksOf('warning')]))
+ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()
+ const check = { key: 'internal_users', filter_to_add: rule } as InsightDataCheckApi
+
+ await expectLogic(logic, () => {
+ logic.actions.excludeFromInsight(check)
+ logic.actions.excludeFromInsight(check)
+ }).toFinishAllListeners()
+
+ expect(vizData.values.querySource?.properties).toEqual([rule])
+ expect(sentSources.at(-1)?.properties).toEqual([rule])
+ expect(dataChecks).toHaveBeenCalledTimes(2)
+ })
+
it("adds the check's rule to the project's filter, then checks again", async () => {
const existing = teamLogic.values.currentTeam?.test_account_filters ?? []
dataChecks.mockImplementation(answer([200, checksOf('warning')]))
diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts
index b974716ca274..31611fd00da0 100644
--- a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts
+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts
@@ -42,7 +42,15 @@ import type {
InsightDataChecksStatusEnumApi,
} from 'products/product_analytics/frontend/generated/api.schemas'
+import type {
+ PathsQuery,
+ PathsV2Query,
+ RetentionQuery,
+ WebOverviewQuery,
+ WebStatsTableQuery,
+} from '../../../../../frontend/src/queries/schema/schema-general'
import { isFinding, postHogAIPromptOf, removesBotTraffic } from './dataCheckText'
+import { withPropertyFilter } from './withPropertyFilter'
/** The insight query the server checks. The server reads the events from the query this chart runs. */
export interface DataChecksQuery {
@@ -62,6 +70,17 @@ export interface insightDataChecksLogicValues {
query: Node | null // insightDataLogic
hasOverrides: boolean // insightLogic
insightDataLoading: boolean // insightVizDataLogic
+ querySource:
+ | FunnelsQuery
+ | LifecycleQuery
+ | PathsQuery
+ | PathsV2Query
+ | RetentionQuery
+ | StickinessQuery
+ | TrendsQuery
+ | WebOverviewQuery
+ | WebStatsTableQuery
+ | null // insightVizDataLogic
selectedTab: SidePanelTab | null // sidePanelStateLogic
sidePanelOpen: boolean // sidePanelStateLogic
currentTeam: TeamPublicType | TeamType | null // teamLogic
@@ -180,6 +199,9 @@ export interface insightDataChecksLogicActions {
closeLabel: () => {
value: true
}
+ excludeFromInsight: (check: InsightDataCheckApi) => {
+ check: InsightDataCheckApi
+ }
fixWithPostHogAI: (check: InsightDataCheckApi) => {
check: InsightDataCheckApi
}
@@ -262,7 +284,7 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
insightDataLogic(props),
['query'],
insightVizDataLogic(props),
- ['insightDataLoading'],
+ ['insightDataLoading', 'querySource'],
insightLogic(props),
['hasOverrides'],
featureFlagLogic,
@@ -292,6 +314,7 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
check,
}),
fixWithPostHogAI: (check: InsightDataCheckApi) => ({ check }),
+ excludeFromInsight: (check: InsightDataCheckApi) => ({ check }),
reportDataChecksViewed: true,
}),
loaders(({ cache }) => ({
@@ -459,8 +482,9 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
],
labelOpen: [
(s) => [s.status, s.labelHeldOpen],
+ // A finding keeps its label, because the icon alone does not say what the person needs to look at.
(status: InsightDataChecksStatus, labelHeldOpen: boolean): boolean =>
- status === 'checking' || labelHeldOpen,
+ status === 'checking' || status === 'warning' || status === 'problem' || labelHeldOpen,
],
}),
listeners(({ actions, values, cache }) => ({
@@ -513,6 +537,20 @@ export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> =
actions.openSidePanel(SidePanelTab.Max, autoRunMaxPrompt(postHogAIPromptOf(check)))
posthog.capture('insight data check fix applied', { check: check.key, fix: 'posthog_ai' })
},
+ excludeFromInsight: ({ check }) => {
+ // The checks still show the finding until they run again, so a second click would add the rule twice.
+ if (values.status === 'checking') {
+ return
+ }
+ const properties =
+ check.filter_to_add &&
+ withPropertyFilter(values.querySource?.properties, check.filter_to_add as AnyPropertyFilter)
+ if (!properties) {
+ return
+ }
+ actions.updateQuerySource({ properties })
+ posthog.capture('insight data check fix applied', { check: check.key, fix: 'exclude_from_insight' })
+ },
addFilterRule: ({ check }) => {
if (!check.filter_to_add) {
return
diff --git a/products/product_analytics/frontend/insights/dataChecks/withPropertyFilter.test.ts b/products/product_analytics/frontend/insights/dataChecks/withPropertyFilter.test.ts
new file mode 100644
index 000000000000..a4efc5a15c3f
--- /dev/null
+++ b/products/product_analytics/frontend/insights/dataChecks/withPropertyFilter.test.ts
@@ -0,0 +1,72 @@
+import {
+ AnyPropertyFilter,
+ FilterLogicalOperator,
+ PropertyFilterType,
+ PropertyGroupFilter,
+ PropertyOperator,
+} from '~/types'
+
+import { withPropertyFilter } from './withPropertyFilter'
+
+const { And, Or } = FilterLogicalOperator
+
+const filterOn = (key: string): AnyPropertyFilter => ({
+ key,
+ type: PropertyFilterType.Person,
+ operator: PropertyOperator.IsNot,
+ value: ['example'],
+})
+const rule = filterOn('email')
+const a = filterOn('a')
+const b = filterOn('b')
+
+describe('withPropertyFilter', () => {
+ it.each<{
+ name: string
+ properties: AnyPropertyFilter[] | PropertyGroupFilter | undefined
+ expected: AnyPropertyFilter[] | PropertyGroupFilter | null
+ }>([
+ { name: 'no filters', properties: undefined, expected: [rule] },
+ { name: 'a list', properties: [a], expected: [a, rule] },
+ {
+ name: 'AND groups',
+ properties: { type: And, values: [{ type: Or, values: [a, b] }] },
+ expected: {
+ type: And,
+ values: [
+ { type: Or, values: [a, b] },
+ { type: And, values: [rule] },
+ ],
+ },
+ },
+ {
+ name: 'OR of AND groups',
+ properties: {
+ type: Or,
+ values: [
+ { type: And, values: [a] },
+ { type: And, values: [b] },
+ ],
+ },
+ expected: {
+ type: Or,
+ values: [
+ { type: And, values: [a, rule] },
+ { type: And, values: [b, rule] },
+ ],
+ },
+ },
+ {
+ name: 'an empty OR group',
+ properties: { type: Or, values: [] },
+ expected: { type: And, values: [{ type: And, values: [rule] }] },
+ },
+ {
+ name: 'OR of an OR group',
+ properties: { type: Or, values: [{ type: Or, values: [a, b] }] },
+ expected: null,
+ },
+ ])('adds the filter to $name without changing what they match', ({ properties, expected }) => {
+ expect(withPropertyFilter(properties, rule)).toEqual(expected)
+ })
+})
diff --git a/products/product_analytics/frontend/insights/dataChecks/withPropertyFilter.ts b/products/product_analytics/frontend/insights/dataChecks/withPropertyFilter.ts
new file mode 100644
index 000000000000..bdc5e089888e
--- /dev/null
+++ b/products/product_analytics/frontend/insights/dataChecks/withPropertyFilter.ts
@@ -0,0 +1,27 @@
+import { isPropertyGroup } from 'lib/components/PropertyFilters/utils'
+
+import { AnyPropertyFilter, FilterLogicalOperator, PropertyGroupFilter } from '~/types'
+
+// Returns null when the filter would need a third level of groups, which the filter editor cannot show.
+export function withPropertyFilter(
+ properties: AnyPropertyFilter[] | PropertyGroupFilter | undefined,
+ filter: AnyPropertyFilter
+): AnyPropertyFilter[] | PropertyGroupFilter | null {
+ if (!isPropertyGroup(properties)) {
+ return [...(properties ?? []), filter]
+ }
+ if (properties.type === FilterLogicalOperator.And || properties.values.length === 0) {
+ return {
+ type: FilterLogicalOperator.And,
+ values: [...properties.values, { type: FilterLogicalOperator.And, values: [filter] }],
+ }
+ }
+ // (A or B) and filter is (A and filter) or (B and filter), so the filter goes into each AND group.
+ if (properties.values.every((group) => group.type === FilterLogicalOperator.And)) {
+ return {
+ ...properties,
+ values: properties.values.map((group) => ({ ...group, values: [...group.values, filter] })),
+ }
+ }
+ return null
+}