Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(web-analytics): extract search and ai query builders (#123196)

Jordan Mryyan724ae135891f
diff --git a/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx b/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx
index 81f7cb7cb40e..11868c1021d0 100644
--- a/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx
+++ b/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx
@@ -10,15 +10,14 @@ import { DataTableNode, NodeKind } from '~/queries/schema/schema-general'
import { InsightLogicProps } from '~/types'
import { SearchAndAiLoading } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiLoading'
-import { SearchAndAiQuery } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiQuery'
-
-import { TileId, WEB_ANALYTICS_DEFAULT_QUERY_TAGS } from './common'
import {
PagePerformanceMetric,
buildGoogleSearchConsoleQuery,
- createPagePerformanceInsightProps,
- pagePerformanceLogic,
-} from './pagePerformanceLogic'
+} from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'
+import { SearchAndAiQuery } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiQuery'
+
+import { TileId, WEB_ANALYTICS_DEFAULT_QUERY_TAGS } from './common'
+import { createPagePerformanceInsightProps, pagePerformanceLogic } from './pagePerformanceLogic'
const GSC_DOCS_URL = 'https://posthog.com/docs/cdp/sources/google-search-console'
const GSC_TABLE_NAME = 'search_analytics_by_query_page'
diff --git a/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.ts b/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.ts
index 0cc3a4e152c9..8f9aad0e256c 100644
--- a/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.ts
+++ b/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.ts
@@ -3,16 +3,19 @@ import { dayjs } from 'lib/dayjs'
import { CompareFilter } from '~/queries/schema/schema-general'
import {
- OverviewTotals,
PagePerformanceWindow,
buildGoogleSearchConsoleQuery,
- buildPagePerformanceTableQuery,
formatShare,
+ resolvePagePerformanceWindow,
+} from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'
+
+import {
+ OverviewTotals,
+ buildPagePerformanceTableQuery,
mergePagePerformanceSeries,
parsePagePerformanceOverviewResponse,
resolvePagePerformanceBucket,
resolvePagePerformanceDataState,
- resolvePagePerformanceWindow,
} from './pagePerformanceLogic'
import { DateFilterState } from './webAnalyticsLogic'
diff --git a/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts b/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts
index 4d3a024ed48c..a2ce7caf95ab 100644
--- a/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts
+++ b/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts
@@ -16,8 +16,6 @@ import {
import { subscriptions } from 'kea-subscriptions'
import { dayjs } from 'lib/dayjs'
-import { componentsToDayJs, dateStringToComponents, dateStringToDayJs } from 'lib/utils/dateFilters'
-import { percentage } from 'lib/utils/numbers'
import { isAbortedRequest } from 'lib/utils/requests'
import { pluralize } from 'lib/utils/strings'
import { teamLogic } from 'scenes/teamLogic'
@@ -43,7 +41,7 @@ import {
WebStatsBreakdown,
WebStatsTableQuery,
} from '~/queries/schema/schema-general'
-import { escapeDottedHogQLIdentifier, escapeHogQLString } from '~/queries/utils'
+import { escapeHogQLString } from '~/queries/utils'
import {
ActionType,
BaseMathType,
@@ -55,6 +53,24 @@ import {
TeamType,
} from '~/types'
+import {
+ AI_VIEW,
+ CRAWLER,
+ GOOGLE_VIEW,
+ HUMAN_EVENTS,
+ HUMAN_VIEW,
+ PAGE_PERFORMANCE_EVENTS,
+ PagePerformanceMetric,
+ PagePerformanceWindow,
+ RAW_PATH,
+ buildBreakdownQuery,
+ buildPagePerformancePathExpr,
+ pageCandidatesPredicate,
+ pageKeyExpr,
+ resolvePagePerformanceWindow,
+ windowPredicates,
+} from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'
+
import type { TeamPublicType } from '../../types'
import {
BOT_ANALYTICS_EVENTS,
@@ -69,25 +85,10 @@ import { webAnalyticsLogic } from './webAnalyticsLogic'
import type { DateFilterState } from './webAnalyticsLogic'
import { WebAnalyticsLogicProps } from './webAnalyticsLogicProps'
-const PAGE_PERFORMANCE_EVENTS = "('$pageview', '$screen', '$http_log')"
-
-const HUMAN_EVENTS = "('$pageview', '$screen')"
-
const PAGE_TABLE_LIMIT = 20
const PAGE_CANDIDATE_LIMIT = 200
-const HUMAN_VIEW = "(event = '$pageview' OR event = '$screen') AND NOT `$virt_is_bot`"
-
-const GOOGLE_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'Organic Search' AND session.$entry_referring_domain ILIKE '%google%'`
-
-const AI_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'AI'`
-
-const CRAWLER =
- "`$virt_is_bot` = true AND `$virt_bot_name` != '' AND `$virt_traffic_category` IN ('ai_crawler', 'ai_assistant', 'ai_search')"
-
-export type PagePerformanceMetric = 'llm_referrals' | 'agent_crawls' | 'google_search'
-
export interface PagePerformanceBreakdownState {
page: string
metric: PagePerformanceMetric
@@ -98,14 +99,6 @@ export interface PagePerformanceOrderBy {
direction: 'ASC' | 'DESC'
}
-export interface PagePerformanceWindow {
- currentFrom: dayjs.Dayjs | null
- currentTo: dayjs.Dayjs
- previousFrom: dayjs.Dayjs | null
- previousTo: dayjs.Dayjs | null
- timezone: string
-}
-
export interface OverviewTotals {
visitors: number
visitorsPrevious: number
@@ -240,8 +233,6 @@ export const resolvePagePerformanceDataState = (
}
}
-const tsLiteral = (date: dayjs.Dayjs, timezone: string): string => date.tz(timezone).format("'YYYY-MM-DD HH:mm:ss'")
-
const conversionMatch = (conversionGoal: WebAnalyticsConversionGoal | null): string | null => {
if (conversionGoal && 'actionId' in conversionGoal) {
return `matchesAction(${(conversionGoal as ActionConversionGoal).actionId})`
@@ -252,55 +243,6 @@ const conversionMatch = (conversionGoal: WebAnalyticsConversionGoal | null): str
return null
}
-const RAW_PATH = 'properties.$pathname'
-
-export const buildPagePerformancePathExpr = (
- source: string,
- enabled: boolean,
- filters: { regex?: string; alias?: string }[] | undefined
-): string => {
- if (!enabled || !filters?.length) {
- return source
- }
- let expr = source
- for (const f of filters) {
- if (f.regex && f.alias != null) {
- expr = `replaceRegexpAll(${expr}, ${escapeHogQLString(f.regex)}, ${escapeHogQLString(f.alias)})`
- }
- }
- return expr
-}
-
-export const resolvePagePerformanceWindow = (
- dateFilter: DateFilterState,
- compareFilter: CompareFilter,
- timezone: string,
- now: dayjs.Dayjs = dayjs().tz(timezone)
-): PagePerformanceWindow => {
- const currentFrom =
- dateFilter.dateFrom === 'all'
- ? null
- : (dateStringToDayJs(dateFilter.dateFrom, timezone) ?? now.subtract(7, 'day'))
- const parsedCurrentTo = dateFilter.dateTo ? dateStringToDayJs(dateFilter.dateTo, timezone) : null
- const currentTo =
- parsedCurrentTo && /^\d{4}-\d{2}-\d{2}$/.test(dateFilter.dateTo ?? '')
- ? parsedCurrentTo.endOf('day')
- : (parsedCurrentTo ?? now)
-
- if (!compareFilter.compare || !currentFrom) {
- return { currentFrom, currentTo, previousFrom: null, previousTo: null, timezone }
- }
-
- const spanMs = Math.max(currentTo.diff(currentFrom), 1)
- const compareToComponents = dateStringToComponents(compareFilter.compare_to ?? null)
- const previousFrom = compareToComponents
- ? componentsToDayJs(compareToComponents, currentFrom, timezone)
- : currentFrom.subtract(spanMs, 'millisecond')
- const previousTo = compareToComponents ? previousFrom.add(spanMs, 'millisecond') : currentFrom
-
- return { currentFrom, currentTo, previousFrom, previousTo, timezone }
-}
-
export interface ParsedOverviewResponse {
totals: Record<string, number>
buckets: { bucket: dayjs.Dayjs; values: Record<string, number> }[]
@@ -420,23 +362,6 @@ export const parseMetricCell = (value: unknown): MetricCellValue | null =>
? { current: Number(value[0] ?? 0), previous: Number(value[1] ?? 0) }
: null
-/** Shares below 10% get a decimal, so a page holding 0.4% of site traffic doesn't read as 0%. */
-export const formatShare = (part: number, whole: number): string | null => {
- if (whole <= 0 || part <= 0) {
- return null
- }
- const fraction = part / whole
- return percentage(fraction, fraction < 0.1 ? 1 : 0)
-}
-
-/** Percent change against the previous period, or null when there is no comparable baseline. */
-export const changeVsPrevious = (current: number, previous: number): number | null => {
- if (previous === 0) {
- return null
- }
- return current === previous ? 0 : current / previous - 1
-}
-
const SORTABLE_COLUMNS = new Set([
'breakdown_value',
'visitors',
@@ -449,23 +374,6 @@ const SORTABLE_COLUMNS = new Set([
const orderByExpr = (column: string): string =>
`"context.columns.${SORTABLE_COLUMNS.has(column) ? column : 'visitors'}"`
-const windowPredicates = (window: PagePerformanceWindow): { cur: string; prev: string; full: string } => {
- const upperBound = `timestamp <= ${tsLiteral(window.currentTo, window.timezone)}`
- const cur = window.currentFrom
- ? `timestamp >= ${tsLiteral(window.currentFrom, window.timezone)} AND ${upperBound}`
- : upperBound
- const prev =
- window.previousFrom && window.previousTo
- ? `timestamp >= ${tsLiteral(window.previousFrom, window.timezone)} AND timestamp < ${tsLiteral(window.previousTo, window.timezone)}`
- : 'false'
- return { cur, prev, full: `(${cur}) OR (${prev})` }
-}
-
-const pageKeyExpr = (pathExpr: string): string => `concat(coalesce(properties.$host, ''), ${pathExpr})`
-
-const pageCandidatesPredicate = (pageKey: string, candidates: string[]): string =>
- candidates.length > 0 ? `${pageKey} IN (${candidates.map(escapeHogQLString).join(', ')})` : 'false'
-
/**
* In the page_durations subquery, `{filters}` matches the carrier event (the NEXT pageview, or the
* pageleave), not the measured page, so an event property that varies per pageview (e.g. $pathname)
@@ -596,67 +504,6 @@ GROUP BY GROUPING SETS ((bucket), ())
`
}
-const buildBreakdownQuery = (
- metric: PagePerformanceMetric,
- page: string,
- window: PagePerformanceWindow,
- pathExpr: string
-): string | null => {
- const { cur } = windowPredicates(window)
- const pageMatch = `${pageKeyExpr(pathExpr)} = ${escapeHogQLString(page)}`
-
- if (metric === 'llm_referrals') {
- return `
-SELECT
- coalesce(nullIf(session.$entry_referring_domain, ''), '(direct)') AS "Assistant",
- uniq(person_id) AS "Visitors"
-FROM events
-WHERE and(${AI_VIEW}, ${pageMatch}, (${cur}), {filters})
-GROUP BY "Assistant"
-ORDER BY "Visitors" DESC
-LIMIT 10
-`
- }
- if (metric === 'agent_crawls') {
- return `
-SELECT
- \`$virt_bot_name\` AS "Crawler",
- \`$virt_traffic_category\` AS "Category",
- count() AS "Requests",
- max(timestamp) AS "Last seen"
-FROM events
-WHERE and(${CRAWLER}, event IN ${PAGE_PERFORMANCE_EVENTS}, ${pageMatch}, (${cur}), {filters})
-GROUP BY "Crawler", "Category"
-ORDER BY "Requests" DESC
-LIMIT 50
-`
- }
- return null
-}
-
-export const buildGoogleSearchConsoleQuery = (
- tableName: string,
- page: string,
- window: PagePerformanceWindow,
- isPathCleaningEnabled: boolean,
- pathCleaningFilters: { regex?: string; alias?: string }[] | undefined
-): string => {
- const gscPath = buildPagePerformancePathExpr('path(page)', isPathCleaningEnabled, pathCleaningFilters)
- const gscPageKey = `concat(domain(page), ${gscPath})`
- const dateField = 'toDate(date)'
- const dateUpperBound = `${dateField} <= toDate(${tsLiteral(window.currentTo, window.timezone)})`
- const datePredicate = window.currentFrom
- ? `${dateField} >= toDate(${tsLiteral(window.currentFrom, window.timezone)}) AND ${dateUpperBound}`
- : dateUpperBound
-
- return `SELECT query AS "Query", sum(clicks) AS "Clicks", round(avg(position), 1) AS "Avg. position"
-FROM ${escapeDottedHogQLIdentifier(tableName)}
-WHERE ${gscPageKey} = ${escapeHogQLString(page)} AND (${datePredicate})
-GROUP BY query
-ORDER BY sum(clicks) DESC
-LIMIT 20`
-}
-
export interface pagePerformanceLogicValues {
allActions: ActionType[]
currentTeam: TeamPublicType | TeamType | null
diff --git a/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsx b/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsx
index d9063defe76f..6deb719a6eaf 100644
--- a/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsx
+++ b/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsx
@@ -8,13 +8,9 @@ import { IconTrendingDown, IconTrendingFlat } from 'lib/lemon-ui/icons'
import { humanFriendlyDuration } from 'lib/utils/durations'
import { percentage } from 'lib/utils/numbers'
import { pluralize } from 'lib/utils/strings'
-import {
- changeVsPrevious,
- formatShare,
- PagePerformanceMetric,
- pagePerformanceLogic,
- parseMetricCell,
-} from 'scenes/web-analytics/pagePerformanceLogic'
+import { pagePerformanceLogic, parseMetricCell } from 'scenes/web-analytics/pagePerformanceLogic'
+
+import { PagePerformanceMetric, changeVsPrevious, formatShare } from './searchAndAiQueries'
export type SearchAndAiMetric =
| 'visitors'
diff --git a/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts b/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts
new file mode 100644
index 000000000000..d2a06b28384a
--- /dev/null
+++ b/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts
@@ -0,0 +1,177 @@
+import { dayjs } from 'lib/dayjs'
+import { componentsToDayJs, dateStringToComponents, dateStringToDayJs } from 'lib/utils/dateFilters'
+import { percentage } from 'lib/utils/numbers'
+import type { DateFilterState } from 'scenes/web-analytics/webAnalyticsLogic'
+
+import { CompareFilter } from '~/queries/schema/schema-general'
+import { escapeDottedHogQLIdentifier, escapeHogQLString } from '~/queries/utils'
+
+export const PAGE_PERFORMANCE_EVENTS = "('$pageview', '$screen', '$http_log')"
+
+export const HUMAN_EVENTS = "('$pageview', '$screen')"
+
+export const HUMAN_VIEW = "(event = '$pageview' OR event = '$screen') AND NOT `$virt_is_bot`"
+
+export const GOOGLE_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'Organic Search' AND session.$entry_referring_domain ILIKE '%google%'`
+
+export const AI_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'AI'`
+
+export const CRAWLER =
+ "`$virt_is_bot` = true AND `$virt_bot_name` != '' AND `$virt_traffic_category` IN ('ai_crawler', 'ai_assistant', 'ai_search')"
+
+export type PagePerformanceMetric = 'llm_referrals' | 'agent_crawls' | 'google_search'
+
+export interface PagePerformanceWindow {
+ currentFrom: dayjs.Dayjs | null
+ currentTo: dayjs.Dayjs
+ previousFrom: dayjs.Dayjs | null
+ previousTo: dayjs.Dayjs | null
+ timezone: string
+}
+
+export const tsLiteral = (date: dayjs.Dayjs, timezone: string): string =>
+ date.tz(timezone).format("'YYYY-MM-DD HH:mm:ss'")
+
+export const RAW_PATH = 'properties.$pathname'
+
+export const buildPagePerformancePathExpr = (
+ source: string,
+ enabled: boolean,
+ filters: { regex?: string; alias?: string }[] | undefined
+): string => {
+ if (!enabled || !filters?.length) {
+ return source
+ }
+ let expr = source
+ for (const f of filters) {
+ if (f.regex && f.alias != null) {
+ expr = `replaceRegexpAll(${expr}, ${escapeHogQLString(f.regex)}, ${escapeHogQLString(f.alias)})`
+ }
+ }
+ return expr
+}
+
+export const resolvePagePerformanceWindow = (
+ dateFilter: DateFilterState,
+ compareFilter: CompareFilter,
+ timezone: string,
+ now: dayjs.Dayjs = dayjs().tz(timezone)
+): PagePerformanceWindow => {
+ const currentFrom =
+ dateFilter.dateFrom === 'all'
+ ? null
+ : (dateStringToDayJs(dateFilter.dateFrom, timezone) ?? now.subtract(7, 'day'))
+ const parsedCurrentTo = dateFilter.dateTo ? dateStringToDayJs(dateFilter.dateTo, timezone) : null
+ const currentTo =
+ parsedCurrentTo && /^\d{4}-\d{2}-\d{2}$/.test(dateFilter.dateTo ?? '')
+ ? parsedCurrentTo.endOf('day')
+ : (parsedCurrentTo ?? now)
+
+ if (!compareFilter.compare || !currentFrom) {
+ return { currentFrom, currentTo, previousFrom: null, previousTo: null, timezone }
+ }
+
+ const spanMs = Math.max(currentTo.diff(currentFrom), 1)
+ const compareToComponents = dateStringToComponents(compareFilter.compare_to ?? null)
+ const previousFrom = compareToComponents
+ ? componentsToDayJs(compareToComponents, currentFrom, timezone)
+ : currentFrom.subtract(spanMs, 'millisecond')
+ const previousTo = compareToComponents ? previousFrom.add(spanMs, 'millisecond') : currentFrom
+
+ return { currentFrom, currentTo, previousFrom, previousTo, timezone }
+}
+
+/** Shares below 10% get a decimal, so a page holding 0.4% of site traffic doesn't read as 0%. */
+export const formatShare = (part: number, whole: number): string | null => {
+ if (whole <= 0 || part <= 0) {
+ return null
+ }
+ const fraction = part / whole
+ return percentage(fraction, fraction < 0.1 ? 1 : 0)
+}
+
+/** Percent change against the previous period, or null when there is no comparable baseline. */
+export const changeVsPrevious = (current: number, previous: number): number | null => {
+ if (previous === 0) {
+ return null
+ }
+ return current === previous ? 0 : current / previous - 1
+}
+
+export const windowPredicates = (window: PagePerformanceWindow): { cur: string; prev: string; full: string } => {
+ const upperBound = `timestamp <= ${tsLiteral(window.currentTo, window.timezone)}`
+ const cur = window.currentFrom
+ ? `timestamp >= ${tsLiteral(window.currentFrom, window.timezone)} AND ${upperBound}`
+ : upperBound
+ const prev =
+ window.previousFrom && window.previousTo
+ ? `timestamp >= ${tsLiteral(window.previousFrom, window.timezone)} AND timestamp < ${tsLiteral(window.previousTo, window.timezone)}`
+ : 'false'
+ return { cur, prev, full: `(${cur}) OR (${prev})` }
+}
+
+export const pageKeyExpr = (pathExpr: string): string => `concat(coalesce(properties.$host, ''), ${pathExpr})`
+
+export const pageCandidatesPredicate = (pageKey: string, candidates: string[]): string =>
+ candidates.length > 0 ? `${pageKey} IN (${candidates.map(escapeHogQLString).join(', ')})` : 'false'
+
+export const buildBreakdownQuery = (
+ metric: PagePerformanceMetric,
+ page: string,
+ window: PagePerformanceWindow,
+ pathExpr: string
+): string | null => {
+ const { cur } = windowPredicates(window)
+ const pageMatch = `${pageKeyExpr(pathExpr)} = ${escapeHogQLString(page)}`
+
+ if (metric === 'llm_referrals') {
+ return `
+SELECT
+ coalesce(nullIf(session.$entry_referring_domain, ''), '(direct)') AS "Assistant",
+ uniq(person_id) AS "Visitors"
+FROM events
+WHERE and(${AI_VIEW}, ${pageMatch}, (${cur}), {filters})
+GROUP BY "Assistant"
+ORDER BY "Visitors" DESC
+LIMIT 10
+`
+ }
+ if (metric === 'agent_crawls') {
+ return `
+SELECT
+ \`$virt_bot_name\` AS "Crawler",
+ \`$virt_traffic_category\` AS "Category",
+ count() AS "Requests",
+ max(timestamp) AS "Last seen"
+FROM events
+WHERE and(${CRAWLER}, event IN ${PAGE_PERFORMANCE_EVENTS}, ${pageMatch}, (${cur}), {filters})
+GROUP BY "Crawler", "Category"
+ORDER BY "Requests" DESC
+LIMIT 50
+`
+ }
+ return null
+}
+
+export const buildGoogleSearchConsoleQuery = (
+ tableName: string,
+ page: string,
+ window: PagePerformanceWindow,
+ isPathCleaningEnabled: boolean,
+ pathCleaningFilters: { regex?: string; alias?: string }[] | undefined
+): string => {
+ const gscPath = buildPagePerformancePathExpr('path(page)', isPathCleaningEnabled, pathCleaningFilters)
+ const gscPageKey = `concat(domain(page), ${gscPath})`
+ const dateField = 'toDate(date)'
+ const dateUpperBound = `${dateField} <= toDate(${tsLiteral(window.currentTo, window.timezone)})`
+ const datePredicate = window.currentFrom
+ ? `${dateField} >= toDate(${tsLiteral(window.currentFrom, window.timezone)}) AND ${dateUpperBound}`
+ : dateUpperBound
+
+ return `SELECT query AS "Query", sum(clicks) AS "Clicks", round(avg(position), 1) AS "Avg. position"
+FROM ${escapeDottedHogQLIdentifier(tableName)}
+WHERE ${gscPageKey} = ${escapeHogQLString(page)} AND (${datePredicate})
+GROUP BY query
+ORDER BY sum(clicks) DESC
+LIMIT 20`
+}