:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(insights): add the logic that runs data checks on the insight page (#116843)
Sam Penningtonbedd7a9384d8
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsxindex 900aef924d55..c17bb2db5bdd 100644--- a/frontend/src/lib/constants.tsx+++ b/frontend/src/lib/constants.tsx@@ -382,6 +382,7 @@ export const FEATURE_FLAGS = {INBOX_SLACK_NOTIFICATIONS: 'inbox-slack-notifications', // owner: #team-self-driving, gates the Slack notifications config card in the inboxINBOX_TIME_WINDOW: 'inbox-time-window', // owner: #team-self-driving, gates the created-in window filter on the inbox report listINSIGHT_ALERTS_EMPTY_STATE_EXPERIMENT: 'insight-alerts-empty-state',+ INSIGHT_DATA_CHECKS: 'insight-data-checks', // owner: @sampennington #team-product-analytics, checks the events behind an insight and shows the result on the insight pageINSIGHT_DRAG_TO_ZOOM: 'insight-drag-to-zoom', // owner: @sampennington #team-product-analytics, gates x-axis drag-to-zoom on insight charts (trends today, SQL to follow)INSIGHT_EXPORT_NUDGE: 'insight-export-nudge', // owner: #team-analytics-platform multivariate=control,test, nudges people who just exported an insight toward a recurring subscriptionINSIGHT_SUBSCRIBE_PROMINENT_BUTTON: 'insight-subscribe-prominent-button', // owner: @mattp #team-analytics-platform multivariate=control,testdiff --git a/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts b/frontend/src/queries/nodes/DataNode/dataNodeLogic.tsindex 0491d461c10b..fb9ad4a76c0d 100644--- a/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts+++ b/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts@@ -189,7 +189,7 @@ function sanitizeRefreshType(refresh: unknown): RefreshType | undefined {: undefined}-const concurrencyController = new ConcurrencyController(1)+export const concurrencyController = new ConcurrencyController(1)const accountsTableConcurrencyController = new ConcurrencyController(2)const webAnalyticsConcurrencyController = new ConcurrencyController(6)const webAnalyticsPreAggConcurrencyController = new ConcurrencyController(6)@@ -199,7 +199,7 @@ const notebookConcurrencyController = new ConcurrencyController(4)// The priority queue doesn't keep insertion order for equal priorities, so notebook loads count up instead.let notebookLoadOrder = 0-async function cancelQueryOnServer(teamId: number | null, queryId: string): Promise<void> {+export async function cancelQueryOnServer(teamId: number | null, queryId: string): Promise<void> {try {// nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. No generated function covers this endpoint yet. Find out why the generated client skips it (no schema, no product tag, or excluded from the spec) and fix that first.await api.delete(`api/projects/${teamId}/query/${queryId}/`)diff --git a/products/product_analytics/backend/presentation/insight.py b/products/product_analytics/backend/presentation/insight.pyindex 7b100fd8ce66..1f9b40557817 100644--- a/products/product_analytics/backend/presentation/insight.py+++ b/products/product_analytics/backend/presentation/insight.py@@ -59,7 +59,7 @@ from posthog.clickhouse.cancel import cancel_query_on_clusterfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import AccessMethod, get_query_tag_value, tag_queries, tags_contextfrom posthog.constants import INSIGHT, AvailableFeature-from posthog.errors import ExposedCHQueryError, QueryErrorCategory, classify_query_error+from posthog.errors import CHQueryErrorQueryWasCancelled, ExposedCHQueryError, QueryErrorCategory, classify_query_errorfrom posthog.event_usage import EventSource, get_event_source, get_request_analytics_properties, report_user_actionfrom posthog.exceptions import ClickHouseAtCapacityfrom posthog.exceptions_capture import capture_exception@@ -1724,6 +1724,14 @@ class InsightDataChecksRequestSerializer(serializers.Serializer):"Pass the query of a saved insight, or one that is not saved yet."))+ client_query_id = serializers.CharField(+ required=False,+ max_length=128,+ help_text=(+ "An id for this run, chosen by the client. To stop the queries of the run, send a DELETE request to "+ "`query/{client_query_id}/`."+ ),+ )class InsightDataCheckSerializer(serializers.Serializer):@@ -2584,6 +2592,8 @@ When set, the specified dashboard's filters and date range override will be appl# The scope covers a token. A session has no scopes, and a token's scope does not prove its owner has access.if not self.user_access_control.check_access_level_for_resource("query", "viewer"):raise PermissionDenied("You need query access to check a query that is not saved as an insight.")+ if client_query_id := request.validated_data.get("client_query_id"):+ tag_queries(client_query_id=client_query_id)return self._data_checks_response(request.validated_data["query"])@validated_request(@@ -2616,6 +2626,9 @@ When set, the specified dashboard's filters and date range override will be appluser=cast(User, self.request.user),email_access=self._team_email_access(),)+ except CHQueryErrorQueryWasCancelled:+ # The client cancelled the run, so nothing reads a body. 499 is Nginx's "Client Closed Request".+ return Response(status=499)except (ExposedHogQLError, ExposedCHQueryError, ResolutionError) as error:raise ValidationError(str(error), getattr(error, "code_name", None))return Response(InsightDataChecksSerializer(checks).data)diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.pyindex d2564daa7e21..26abcc59a1d4 100644--- a/products/product_analytics/backend/tests/test_insight_data_checks.py+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py@@ -14,8 +14,9 @@ from django.test import SimpleTestCasefrom parameterized import parameterizedfrom rest_framework import status+from posthog.clickhouse.query_tagging import get_query_tag_valuefrom posthog.constants import AvailableFeature-from posthog.errors import ExposedCHQueryError+from posthog.errors import CHQueryErrorQueryWasCancelled, ExposedCHQueryErrorfrom posthog.exceptions import APIQueriesBudgetExceededfrom posthog.models import PersonalAPIKey, PropertyDefinitionfrom posthog.models.organization import OrganizationMembership@@ -934,3 +935,21 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):response = self.client.post(f"/api/projects/{self.team.id}/insights/data_checks/", body, format="json")assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content++ def test_tags_the_check_queries_with_the_client_query_id_so_the_client_can_cancel_them(self) -> None:+ tagged: list[str | None] = []++ def cancelled(*args: Any, **kwargs: Any) -> None:+ tagged.append(get_query_tag_value("client_query_id"))+ raise CHQueryErrorQueryWasCancelled("Query was cancelled", code=394, code_name="query_was_cancelled")++ with patch(f"{CHECKS_MODULE}.execute_hogql_query", side_effect=cancelled):+ response = self.client.post(+ f"/api/projects/{self.team.id}/insights/data_checks/",+ {"query": self._query(), "client_query_id": "checks-run-1"},+ format="json",+ )++ assert tagged == ["checks-run-1"]+ assert response.status_code == 499, response.content+ assert response.content == b""diff --git a/products/product_analytics/frontend/generated/api.schemas.ts b/products/product_analytics/frontend/generated/api.schemas.tsindex 4aecc3199dc4..24d980277c3a 100644--- a/products/product_analytics/frontend/generated/api.schemas.ts+++ b/products/product_analytics/frontend/generated/api.schemas.ts@@ -9492,6 +9492,11 @@ export interface BulkUpdateTagsResponseApi {export interface InsightDataChecksRequestApi {/** The insight query to check, in the shape an insight stores it: an `InsightVizNode` with its `source`. Pass the query of a saved insight, or one that is not saved yet. */query: _InsightQuerySchemaApi+ /**+ * An id for this run, chosen by the client. To stop the queries of the run, send a DELETE request to `query/{client_query_id}/`.+ * @maxLength 128+ */+ client_query_id?: string}/**diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.tsnew file mode 100644index 000000000000..306443852009--- /dev/null+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.test.ts@@ -0,0 +1,273 @@+import { expectLogic } from 'kea-test-utils'++import { FEATURE_FLAGS } from 'lib/constants'+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'+import { insightDataLogic } from 'scenes/insights/insightDataLogic'+import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic'++import { useMocks } from '~/mocks/jest'+import { NodeKind } from '~/queries/schema/schema-general'+import type { InsightVizNode } from '~/queries/schema/schema-general'+import { initKeaTests } from '~/test/init'+import { ChartDisplayType } from '~/types'+import type { InsightShortId } from '~/types'++import { insightDataChecksLogic } from './insightDataChecksLogic'++const props = { dashboardItemId: '123' as InsightShortId }+const checksOf = (status: string): Record<string, unknown> => ({+ status,+ checks: [],+ coverage: null,+})++describe('insightDataChecksLogic', () => {+ let logic: ReturnType<typeof insightDataChecksLogic.build>+ let vizData: ReturnType<typeof insightVizDataLogic.build>+ const dataChecks = jest.fn()+ const cancelledQueryIds: string[] = []+ let sentSources: Record<string, unknown>[]++ const answer =+ (response: [number, Record<string, unknown>], held?: Promise<void>, onSent?: () => void) =>+ async ({ request }: { request: Request }): Promise<[number, Record<string, unknown>]> => {+ const body = (await request.json()) as { query: { source: Record<string, unknown> } }+ sentSources.push(body.query.source)+ onSent?.()+ await held+ return response+ }++ const turnTheFlagOn = (): void => {+ featureFlagLogic.actions.setFeatureFlags([FEATURE_FLAGS.INSIGHT_DATA_CHECKS], {+ [FEATURE_FLAGS.INSIGHT_DATA_CHECKS]: true,+ })+ }++ beforeEach(() => {+ dataChecks.mockReset()+ sentSources = []+ cancelledQueryIds.length = 0+ useMocks({+ get: {+ '/api/environments/:team_id/insights/trend': [],+ '/api/environments/:team_id/insights/': { results: [{}] },+ },+ post: {+ '/api/environments/:team_id/query/:kind': { results: [] },+ '/api/projects/:team_id/insights/data_checks/': dataChecks,+ },+ delete: {+ '/api/projects/:team_id/query/:id/': ({ params }) => {+ cancelledQueryIds.push(params.id as string)+ return [200, {}]+ },+ },+ })+ initKeaTests()+ featureFlagLogic.mount()+ insightDataLogic(props).mount()+ vizData = insightVizDataLogic(props)+ vizData.mount()+ logic = insightDataChecksLogic(props)+ logic.mount()+ })++ afterEach(() => {+ logic.unmount()+ })++ it('checks the insight when the flag arrives after the chart has loaded', async () => {+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))+ await expectLogic(logic).toFinishAllListeners()+ expect(dataChecks).not.toHaveBeenCalled()++ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()++ expect(dataChecks).toHaveBeenCalledTimes(1)+ expect(logic.values.status).toBe('passed')+ })++ const holdTheFirstAnswer = (): { sent: Promise<void>; release: () => void } => {+ let markSent: () => void = () => {}+ let release: () => void = () => {}+ const sent = new Promise<void>((resolve) => {+ markSent = resolve+ })+ const held = new Promise<void>((resolve) => {+ release = resolve+ })+ dataChecks.mockImplementationOnce(answer([200, checksOf('passed')], held, markSent))+ dataChecks.mockImplementation(answer([200, checksOf('warning')]))+ return { sent, release: () => release() }+ }++ it('runs one check at a time, then checks the latest query', async () => {+ const first = holdTheFirstAnswer()+ turnTheFlagOn()+ await first.sent++ const firstQuery = logic.values.requestedQuery++ vizData.actions.updateQuerySource({ filterTestAccounts: true })+ logic.actions.checkData()+ vizData.actions.updateQuerySource({ dateRange: { date_from: '-30d' } })+ logic.actions.checkData()++ expect(logic.values.requestedQuery).toBe(firstQuery)+ expect(logic.values.status).toBe('checking')++ await expectLogic(logic, first.release).toFinishAllListeners()++ expect(sentSources).toEqual([+ expect.not.objectContaining({ filterTestAccounts: true }),+ expect.objectContaining({ filterTestAccounts: true, dateRange: { date_from: '-30d' } }),+ ])+ expect(logic.values.status).toBe('warning')+ })++ it('sends the whole query the chart runs, so the server reads the same events', async () => {+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))+ const onChrome = { key: '$browser', value: ['Chrome'], operator: 'exact', type: 'event' }+ const modifiers = { personsOnEventsMode: 'person_id_override_properties_joined' }+ const source = {+ kind: NodeKind.FunnelsQuery,+ series: [+ {+ kind: NodeKind.EventsNode,+ event: 'signed up',+ name: 'Sign up',+ math: 'total',+ properties: [onChrome],+ },+ ],+ dateRange: { date_from: '-30d' },+ filterTestAccounts: true,+ funnelsFilter: { funnelAggregateByHogQL: 'properties.$session_id', funnelVizType: 'steps' },+ breakdownFilter: { breakdown_type: 'event', breakdown: '$os' },+ modifiers,+ }+ insightDataLogic(props).actions.setQuery({+ kind: NodeKind.InsightVizNode,+ source,+ } as InsightVizNode)++ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()++ expect(sentSources.at(-1)).toEqual(source)+ })++ it('checks again after a refresh of the chart that ends during a check', async () => {+ const first = holdTheFirstAnswer()+ turnTheFlagOn()+ await first.sent+ vizData.actions.loadDataSuccess(+ {},+ { refresh: 'force_blocking', queryId: 'chart', pollOnly: false, overrideQuery: undefined }+ )++ await expectLogic(logic, first.release).toFinishAllListeners()++ expect(dataChecks).toHaveBeenCalledTimes(2)+ })++ it.each([+ ['again after a person refreshes the chart', 'force_blocking' as const, 2],+ ['once when the chart loads again for the same query', 'blocking' as const, 1],+ ])('checks %s', async (_name, refresh, requests) => {+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))+ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()++ await expectLogic(logic, () => {+ vizData.actions.loadDataSuccess(+ {},+ { refresh, queryId: 'chart', pollOnly: false, overrideQuery: undefined }+ )+ }).toFinishAllListeners()++ expect(dataChecks).toHaveBeenCalledTimes(requests)+ })++ it.each([+ ['hides the mark from a person who may not run the checks', 403, 'unavailable'],+ ['reports a check the server could not finish', 500, 'failed'],+ ])('%s', async (_name, responseStatus, expected) => {+ dataChecks.mockImplementation(answer([responseStatus, { detail: 'No.' }]))++ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()++ expect(logic.values.status).toBe(expected)+ })++ it('checks again when the chart type changes and the chart reuses its result', async () => {+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))+ await expectLogic(logic, turnTheFlagOn).toFinishAllListeners()++ await expectLogic(logic, () => {+ vizData.actions.updateQuerySource({ trendsFilter: { display: ChartDisplayType.ActionsBar } })+ }).toFinishAllListeners()++ expect(dataChecks).toHaveBeenCalledTimes(2)+ expect(logic.values.status).toBe('passed')+ })++ it('does not check a chart with overrides, because only the server holds the query it runs', async () => {+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))+ const overridden = { dashboardItemId: '456' as InsightShortId, filtersOverride: { date_from: '-1d' } }+ insightDataLogic(overridden).mount()+ insightVizDataLogic(overridden).mount()+ const overriddenLogic = insightDataChecksLogic(overridden)+ overriddenLogic.mount()++ await expectLogic(overriddenLogic, turnTheFlagOn).toFinishAllListeners()++ expect(overriddenLogic.values.status).toBe('unavailable')+ expect(dataChecks).toHaveBeenCalledTimes(1)+ overriddenLogic.unmount()+ })++ it('cancels a running check when the insight unmounts, so it frees the chart queue', async () => {+ let signal: AbortSignal | undefined+ let markSent: () => void = () => {}+ const sent = new Promise<void>((resolve) => {+ markSent = resolve+ })+ dataChecks.mockImplementation(async ({ request }: { request: Request }) => {+ signal = request.signal+ markSent()+ return await new Promise(() => {})+ })+ turnTheFlagOn()+ await sent++ logic.unmount()++ expect(signal?.aborted).toBe(true)+ logic.mount()+ })++ it('gives the chart queue back when the chart starts to load during a check, then checks again', async () => {+ let signal: AbortSignal | undefined+ let clientQueryId: string | undefined+ let markSent: () => void = () => {}+ const sent = new Promise<void>((resolve) => {+ markSent = resolve+ })+ dataChecks.mockImplementationOnce(async ({ request }: { request: Request }) => {+ signal = request.signal+ clientQueryId = ((await request.json()) as { client_query_id: string }).client_query_id+ markSent()+ return await new Promise(() => {})+ })+ dataChecks.mockImplementation(answer([200, checksOf('passed')]))+ turnTheFlagOn()+ await sent++ await expectLogic(logic, () => vizData.actions.loadData()).toFinishAllListeners()++ expect(signal?.aborted).toBe(true)+ expect(cancelledQueryIds).toEqual([clientQueryId])+ expect(dataChecks).toHaveBeenCalledTimes(2)+ expect(logic.values.status).toBe('passed')+ })+})diff --git a/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.tsnew file mode 100644index 000000000000..b8513e3dd1b9--- /dev/null+++ b/products/product_analytics/frontend/insights/dataChecks/insightDataChecksLogic.ts@@ -0,0 +1,432 @@+import { MakeLogicType, actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'+import type { LogicWrapper } from 'kea'+import { loaders } from 'kea-loaders'+import { subscriptions } from 'kea-subscriptions'+import posthog from 'posthog-js'++import { ApiError } from 'lib/api-error'+import { FEATURE_FLAGS } from 'lib/constants'+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'+import type { FeatureFlagsSet } from 'lib/logic/featureFlagLogic'+import { uuid } from 'lib/utils/dom'+import { getCurrentTeamId } from 'lib/utils/getAppContext'+import { objectsEqual } from 'lib/utils/objects'+import { insightDataLogic } from 'scenes/insights/insightDataLogic'+import { insightLogic } from 'scenes/insights/insightLogic'+import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic'+import { keyForInsightLogicProps } from 'scenes/insights/sharedUtils'++import { cancelQueryOnServer, concurrencyController } from '~/queries/nodes/DataNode/dataNodeLogic'+import { NodeKind } from '~/queries/schema/schema-general'+import type {+ FunnelsQuery,+ LifecycleQuery,+ Node,+ RefreshType,+ StickinessQuery,+ TrendsQuery,+} from '~/queries/schema/schema-general'+import { isEventsNode, isInsightQueryWithSeries, isInsightVizNode } from '~/queries/utils'+import type { InsightLogicProps } from '~/types'++import { insightsDataChecksCreate } from 'products/product_analytics/frontend/generated/api'+import type {+ InsightDataCheckApi,+ InsightDataChecksApi,+ InsightDataChecksRequestApi,+ InsightDataChecksStatusEnumApi,+} from 'products/product_analytics/frontend/generated/api.schemas'++/** The insight query the server checks. The server reads the events from the query this chart runs. */+export interface DataChecksQuery {+ kind: NodeKind.InsightVizNode+ source: TrendsQuery | FunnelsQuery | StickinessQuery | LifecycleQuery+}++// A failed run is a value and not a loader error, so a check that fails in the background shows no error toast.+export type DataChecksRun = { ok: true; result: InsightDataChecksApi } | { ok: false; denied: boolean }++/** `unavailable`: the checks do not apply here. `checking`: the chart or the checks are still loading. */+export type InsightDataChecksStatus = 'unavailable' | 'checking' | 'failed' | InsightDataChecksStatusEnumApi++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface insightDataChecksLogicValues {+ featureFlags: FeatureFlagsSet // featureFlagLogic+ query: Node | null // insightDataLogic+ hasOverrides: boolean // insightLogic+ insightDataLoading: boolean // insightVizDataLogic+ checkAgainRequested: boolean+ checkedQuery: DataChecksQuery | null+ checks: InsightDataCheckApi[]+ dataChecksRun: DataChecksRun | null+ dataChecksRunLoading: boolean+ filtersTestAccounts: boolean+ findings: InsightDataCheckApi[]+ labelHeldOpen: boolean+ labelOpen: boolean+ requestedQuery: DataChecksQuery | null+ result: InsightDataChecksApi | null+ status: InsightDataChecksStatus+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface insightDataChecksLogicActions {+ setFeatureFlags: (+ flags: string[],+ variants: Record<string, boolean | string>+ ) => {+ flags: string[]+ variants: Record<string, boolean | string>+ } // featureFlagLogic+ loadData: (+ refresh?: RefreshType | undefined,+ alreadyRunningQueryId?: string | undefined,+ overrideQuery?: import('~/queries/schema').DataNode<Record<string, any>> | undefined+ ) => {+ overrideQuery: import('~/queries/schema').DataNode<Record<string, any>> | undefined+ pollOnly: boolean+ queryId: string+ refresh: RefreshType | undefined+ } // insightVizDataLogic+ loadDataSuccess: (+ response:+ | Record<string, any>+ | null+ | import('~/queries/schema').ErrorTrackingQueryResponse+ | import('~/queries/schema').HogQLAutocompleteResponse+ | import('~/queries/schema').HogQLMetadataResponse+ | import('~/queries/schema').HogQLQueryResponse<any[]>+ | import('~/queries/schema').HogQueryResponse+ | import('~/queries/schema').LogAttributesQueryResponse+ | import('~/queries/schema').LogValuesQueryResponse+ | import('~/queries/schema').MetricsQueryResponse+ | import('~/queries/schema').SessionsQueryResponse+ | import('~/queries/schema').TraceSpansAggregationQueryResponse+ | import('~/queries/schema').TraceSpansAttributeBreakdownQueryResponse+ | import('~/queries/schema').TraceSpansQueryResponse+ | import('~/queries/schema').TraceSpansTreeQueryResponse,+ payload?:+ | {+ overrideQuery: import('~/queries/schema').DataNode<Record<string, any>> | undefined+ pollOnly: boolean+ queryId: string+ refresh: RefreshType | undefined+ }+ | undefined+ ) => {+ payload?: {+ overrideQuery: import('~/queries/schema').DataNode<Record<string, any>> | undefined+ pollOnly: boolean+ queryId: string+ refresh: RefreshType | undefined+ }+ response:+ | Record<string, any>+ | null+ | import('~/queries/schema').ErrorTrackingQueryResponse+ | import('~/queries/schema').HogQLAutocompleteResponse+ | import('~/queries/schema').HogQLMetadataResponse+ | import('~/queries/schema').HogQLQueryResponse<any[]>+ | import('~/queries/schema').HogQueryResponse+ | import('~/queries/schema').LogAttributesQueryResponse+ | import('~/queries/schema').LogValuesQueryResponse+ | import('~/queries/schema').MetricsQueryResponse+ | import('~/queries/schema').SessionsQueryResponse+ | import('~/queries/schema').TraceSpansAggregationQueryResponse+ | import('~/queries/schema').TraceSpansAttributeBreakdownQueryResponse+ | import('~/queries/schema').TraceSpansQueryResponse+ | import('~/queries/schema').TraceSpansTreeQueryResponse+ } // insightVizDataLogic+ updateQuerySource: (querySource: import('scenes/insights/insightVizDataLogic').QuerySourceUpdate) => {+ querySource: import('scenes/insights/insightVizDataLogic').QuerySourceUpdate+ } // insightVizDataLogic+ checkData: (again?: boolean) => {+ again: boolean+ }+ closeLabel: () => {+ value: true+ }+ filterOutInternalUsers: () => {+ value: true+ }+ loadDataChecks: (query: DataChecksQuery) => DataChecksQuery+ loadDataChecksFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadDataChecksSuccess: (+ dataChecksRun: DataChecksRun | null,+ payload?: DataChecksQuery+ ) => {+ dataChecksRun: DataChecksRun | null+ payload?: DataChecksQuery+ }+ reportDataChecksViewed: () => {+ value: true+ }+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface insightDataChecksLogicMeta {+ key: string+ __keaTypeGenInternalSelectorTypes: {+ checkedQuery: (+ query: Node<Record<string, any>> | null,+ featureFlags: FeatureFlagsSet,+ hasOverrides: boolean+ ) => DataChecksQuery | null+ status: (+ checkedQuery: DataChecksQuery | null,+ requestedQuery: DataChecksQuery | null,+ dataChecksRunLoading: boolean,+ dataChecksRun: DataChecksRun | null+ ) => InsightDataChecksStatus+ result: (dataChecksRun: DataChecksRun | null, status: InsightDataChecksStatus) => InsightDataChecksApi | null+ filtersTestAccounts: (result: InsightDataChecksApi | null) => boolean+ checks: (result: InsightDataChecksApi | null) => InsightDataCheckApi[]+ findings: (checks: InsightDataCheckApi[]) => InsightDataCheckApi[]+ labelOpen: (status: InsightDataChecksStatus, labelHeldOpen: boolean) => boolean+ }+}++export type insightDataChecksLogicType = MakeLogicType<+ insightDataChecksLogicValues,+ insightDataChecksLogicActions,+ InsightLogicProps,+ insightDataChecksLogicMeta+>++const FORCED_REFRESHES: (RefreshType | undefined)[] = ['force_async', 'force_blocking']+// The checks run up to four queries of 20 seconds each, after a wait for server slots.+const DATA_CHECKS_TIMEOUT_MS = 120_000+const LABEL_OPEN_MS = 2500+const YIELDED_TO_CHART = 'yielded to chart'++export const insightDataChecksLogic: LogicWrapper<insightDataChecksLogicType> = kea<insightDataChecksLogicType>([+ props({} as InsightLogicProps),+ key(keyForInsightLogicProps('new')),+ path((key) => ['products', 'product_analytics', 'insights', 'dataChecks', 'insightDataChecksLogic', key]),+ connect((props: InsightLogicProps) => ({+ values: [+ insightDataLogic(props),+ ['query'],+ insightVizDataLogic(props),+ ['insightDataLoading'],+ insightLogic(props),+ ['hasOverrides'],+ featureFlagLogic,+ ['featureFlags'],+ ],+ actions: [+ insightVizDataLogic(props),+ ['loadData', 'loadDataSuccess', 'updateQuerySource'],+ featureFlagLogic,+ ['setFeatureFlags'],+ ],+ })),+ actions({+ checkData: (again: boolean = false) => ({ again }),+ closeLabel: true,+ filterOutInternalUsers: true,+ reportDataChecksViewed: true,+ }),+ loaders(({ cache }) => ({+ dataChecksRun: [+ null as DataChecksRun | null,+ {+ loadDataChecks: async (query: DataChecksQuery): Promise<DataChecksRun | null> => {+ // Leaving the insight cancels the run, queued or running, so it does not hold the chart queue.+ // A hidden tab keeps it, because a person comes back to the result.+ const abortController = new AbortController()+ const clientQueryId = uuid()+ let sent = false+ cache.dataChecksAbortController = abortController+ cache.disposables.add(() => () => abortController.abort(), 'dataChecksRun', {+ pauseOnPageHidden: false,+ })+ try {+ // The checks hold the same server slots as charts, so they wait behind the charts in the queue.+ const result = await concurrencyController.run({+ debugTag: 'InsightDataChecks',+ abortController,+ fn: async () => {+ // A stalled request would block every later run, which waits for this one to end.+ const timeout = setTimeout(() => abortController.abort(), DATA_CHECKS_TIMEOUT_MS)+ sent = true+ try {+ return await insightsDataChecksCreate(+ String(getCurrentTeamId()),+ {+ query,+ client_query_id: clientQueryId,+ } as unknown as InsightDataChecksRequestApi,+ undefined,+ { signal: abortController.signal }+ )+ } finally {+ clearTimeout(timeout)+ }+ },+ })+ return { ok: true, result }+ } catch (error) {+ // An aborted request does not stop the queries the server runs for it, and they hold its slots.+ if (sent && abortController.signal.aborted) {+ void cancelQueryOnServer(getCurrentTeamId(), clientQueryId)+ }+ if (abortController.signal.reason === YIELDED_TO_CHART) {+ return null+ }+ return { ok: false, denied: error instanceof ApiError && error.status === 403 }+ }+ },+ },+ ],+ })),+ reducers({+ requestedQuery: [+ null as DataChecksQuery | null,+ {+ loadDataChecks: (_, query) => query,+ },+ ],+ // A person can refresh the chart during a run. That run is for the events as they were before.+ checkAgainRequested: [+ false,+ {+ checkData: (requested, { again }) => requested || again,+ loadDataChecks: () => false,+ },+ ],+ labelHeldOpen: [+ false,+ {+ loadDataChecksSuccess: () => true,+ closeLabel: () => false,+ },+ ],+ }),+ selectors({+ // The server only checks insights whose series are all events, so no request goes out for the rest.+ // Compared by value, so a render that rebuilds the same query is not a new query.+ checkedQuery: [+ (s) => [s.query, s.featureFlags, s.hasOverrides],+ (query: Node | null, featureFlags: FeatureFlagsSet, hasOverrides: boolean): DataChecksQuery | null => {+ // The server applies a chart's overrides, so the page does not hold the query the chart runs.+ if (!featureFlags[FEATURE_FLAGS.INSIGHT_DATA_CHECKS] || !isInsightVizNode(query) || hasOverrides) {+ return null+ }+ const source = query.source+ if (!isInsightQueryWithSeries(source) || !source.series.length || !source.series.every(isEventsNode)) {+ return null+ }+ return { kind: NodeKind.InsightVizNode, source }+ },+ { resultEqualityCheck: objectsEqual },+ ],+ status: [+ (s) => [s.checkedQuery, s.requestedQuery, s.dataChecksRunLoading, s.dataChecksRun],+ (+ checkedQuery: DataChecksQuery | null,+ requestedQuery: DataChecksQuery | null,+ dataChecksRunLoading: boolean,+ dataChecksRun: DataChecksRun | null+ ): InsightDataChecksStatus => {+ if (!checkedQuery) {+ return 'unavailable'+ }+ // The checks run after the chart loads, so the last run is out of date from the moment the query changes.+ if (dataChecksRunLoading || !dataChecksRun || requestedQuery !== checkedQuery) {+ return 'checking'+ }+ if (dataChecksRun.ok) {+ return dataChecksRun.result.status+ }+ return dataChecksRun.denied ? 'unavailable' : 'failed'+ },+ ],+ result: [+ (s) => [s.dataChecksRun, s.status],+ (dataChecksRun: DataChecksRun | null, status: InsightDataChecksStatus): InsightDataChecksApi | null =>+ dataChecksRun?.ok && status !== 'checking' ? dataChecksRun.result : null,+ ],+ filtersTestAccounts: [+ (s) => [s.result],+ (result: InsightDataChecksApi | null): boolean => result?.coverage?.filters_test_accounts ?? false,+ ],+ checks: [+ (s) => [s.result],+ (result: InsightDataChecksApi | null): InsightDataCheckApi[] => result?.checks ?? [],+ ],+ findings: [+ (s) => [s.checks],+ (checks: InsightDataCheckApi[]): InsightDataCheckApi[] =>+ checks.filter((check) => check.level === 'warning' || check.level === 'problem'),+ ],+ labelOpen: [+ (s) => [s.status, s.labelHeldOpen],+ (status: InsightDataChecksStatus, labelHeldOpen: boolean): boolean =>+ status === 'checking' || labelHeldOpen,+ ],+ }),+ listeners(({ actions, values, cache }) => ({+ // The chart can load more than once for one query, and the answer for that query is already on its way.+ // One run at a time: a run for an older query keeps its queries on the server until it ends.+ checkData: () => {+ const query = values.checkedQuery+ const isWanted = values.checkAgainRequested || query !== values.requestedQuery+ if (query && isWanted && !values.dataChecksRunLoading) {+ actions.loadDataChecks(query)+ }+ },+ // The query can change during a run. The chart for the new query can finish before that run does.+ loadDataChecksSuccess: async (_, breakpoint) => {+ if (!values.insightDataLoading) {+ actions.checkData()+ }+ await breakpoint(LABEL_OPEN_MS)+ actions.closeLabel()+ },+ // The flags can arrive after the chart has loaded.+ setFeatureFlags: () => {+ if (!values.insightDataLoading) {+ actions.checkData()+ }+ },+ // The queue has one slot, so a check in flight would hold back the chart. The check runs again after the chart.+ loadData: ({ pollOnly }) => {+ if (!pollOnly && values.dataChecksRunLoading) {+ actions.checkData(true)+ cache.dataChecksAbortController?.abort(YIELDED_TO_CHART)+ }+ },+ // A person refreshes the chart after a fix in the project's filter, which leaves the query as it was.+ loadDataSuccess: ({ payload }) => {+ actions.checkData(FORCED_REFRESHES.includes(payload?.refresh))+ },+ filterOutInternalUsers: () => {+ actions.updateQuerySource({ filterTestAccounts: true })+ // pinned: analytics event name, renaming breaks dashboards+ posthog.capture('insight data check fix applied', { check: 'internal_users' })+ },+ reportDataChecksViewed: () => {+ // pinned: analytics event name, renaming breaks dashboards+ posthog.capture('insight data checks viewed', {+ status: values.status,+ findings: values.findings.map((check) => check.key),+ })+ },+ })),+ // Also on mount, because the chart can finish loading before this logic mounts. A change such as the chart+ // type can reuse the chart's result, so no chart load follows it.+ subscriptions(({ actions, values }) => ({+ checkedQuery: () => {+ if (!values.insightDataLoading) {+ actions.checkData()+ }+ },+ })),+])diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex a13c0b9d752d..0e2a84c1c8b9 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -60893,6 +60893,11 @@ export namespace Schemas {export interface InsightDataChecksRequest {/** The insight query to check, in the shape an insight stores it: an `InsightVizNode` with its `source`. Pass the query of a saved insight, or one that is not saved yet. */query: _InsightQuerySchema;+ /**+ * An id for this run, chosen by the client. To stop the queries of the run, send a DELETE request to `query/{client_query_id}/`.+ * @maxLength 128+ */+ client_query_id?: string;}/**