:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
refactor(web-analytics): extract search and ai query builders (#123196)
Jordan Mryyan724ae135891f
diff --git a/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx b/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsxindex 81f7cb7cb40e..11868c1021d0 100644--- a/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx+++ b/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx@@ -10,15 +10,14 @@ import { DataTableNode, NodeKind } from '~/queries/schema/schema-general'import { InsightLogicProps } from '~/types'import { SearchAndAiLoading } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiLoading'-import { SearchAndAiQuery } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiQuery'--import { TileId, WEB_ANALYTICS_DEFAULT_QUERY_TAGS } from './common'import {PagePerformanceMetric,buildGoogleSearchConsoleQuery,- createPagePerformanceInsightProps,- pagePerformanceLogic,-} from './pagePerformanceLogic'+} from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'+import { SearchAndAiQuery } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiQuery'++import { TileId, WEB_ANALYTICS_DEFAULT_QUERY_TAGS } from './common'+import { createPagePerformanceInsightProps, pagePerformanceLogic } from './pagePerformanceLogic'const GSC_DOCS_URL = 'https://posthog.com/docs/cdp/sources/google-search-console'const GSC_TABLE_NAME = 'search_analytics_by_query_page'diff --git a/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.ts b/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.tsindex 0cc3a4e152c9..8f9aad0e256c 100644--- a/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.ts+++ b/frontend/src/scenes/web-analytics/pagePerformanceLogic.test.ts@@ -3,16 +3,19 @@ import { dayjs } from 'lib/dayjs'import { CompareFilter } from '~/queries/schema/schema-general'import {- OverviewTotals,PagePerformanceWindow,buildGoogleSearchConsoleQuery,- buildPagePerformanceTableQuery,formatShare,+ resolvePagePerformanceWindow,+} from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'++import {+ OverviewTotals,+ buildPagePerformanceTableQuery,mergePagePerformanceSeries,parsePagePerformanceOverviewResponse,resolvePagePerformanceBucket,resolvePagePerformanceDataState,- resolvePagePerformanceWindow,} from './pagePerformanceLogic'import { DateFilterState } from './webAnalyticsLogic'diff --git a/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts b/frontend/src/scenes/web-analytics/pagePerformanceLogic.tsindex 4d3a024ed48c..a2ce7caf95ab 100644--- a/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts+++ b/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts@@ -16,8 +16,6 @@ import {import { subscriptions } from 'kea-subscriptions'import { dayjs } from 'lib/dayjs'-import { componentsToDayJs, dateStringToComponents, dateStringToDayJs } from 'lib/utils/dateFilters'-import { percentage } from 'lib/utils/numbers'import { isAbortedRequest } from 'lib/utils/requests'import { pluralize } from 'lib/utils/strings'import { teamLogic } from 'scenes/teamLogic'@@ -43,7 +41,7 @@ import {WebStatsBreakdown,WebStatsTableQuery,} from '~/queries/schema/schema-general'-import { escapeDottedHogQLIdentifier, escapeHogQLString } from '~/queries/utils'+import { escapeHogQLString } from '~/queries/utils'import {ActionType,BaseMathType,@@ -55,6 +53,24 @@ import {TeamType,} from '~/types'+import {+ AI_VIEW,+ CRAWLER,+ GOOGLE_VIEW,+ HUMAN_EVENTS,+ HUMAN_VIEW,+ PAGE_PERFORMANCE_EVENTS,+ PagePerformanceMetric,+ PagePerformanceWindow,+ RAW_PATH,+ buildBreakdownQuery,+ buildPagePerformancePathExpr,+ pageCandidatesPredicate,+ pageKeyExpr,+ resolvePagePerformanceWindow,+ windowPredicates,+} from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'+import type { TeamPublicType } from '../../types'import {BOT_ANALYTICS_EVENTS,@@ -69,25 +85,10 @@ import { webAnalyticsLogic } from './webAnalyticsLogic'import type { DateFilterState } from './webAnalyticsLogic'import { WebAnalyticsLogicProps } from './webAnalyticsLogicProps'-const PAGE_PERFORMANCE_EVENTS = "('$pageview', '$screen', '$http_log')"--const HUMAN_EVENTS = "('$pageview', '$screen')"-const PAGE_TABLE_LIMIT = 20const PAGE_CANDIDATE_LIMIT = 200-const HUMAN_VIEW = "(event = '$pageview' OR event = '$screen') AND NOT `$virt_is_bot`"--const GOOGLE_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'Organic Search' AND session.$entry_referring_domain ILIKE '%google%'`--const AI_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'AI'`--const CRAWLER =- "`$virt_is_bot` = true AND `$virt_bot_name` != '' AND `$virt_traffic_category` IN ('ai_crawler', 'ai_assistant', 'ai_search')"--export type PagePerformanceMetric = 'llm_referrals' | 'agent_crawls' | 'google_search'-export interface PagePerformanceBreakdownState {page: stringmetric: PagePerformanceMetric@@ -98,14 +99,6 @@ export interface PagePerformanceOrderBy {direction: 'ASC' | 'DESC'}-export interface PagePerformanceWindow {- currentFrom: dayjs.Dayjs | null- currentTo: dayjs.Dayjs- previousFrom: dayjs.Dayjs | null- previousTo: dayjs.Dayjs | null- timezone: string-}-export interface OverviewTotals {visitors: numbervisitorsPrevious: number@@ -240,8 +233,6 @@ export const resolvePagePerformanceDataState = (}}-const tsLiteral = (date: dayjs.Dayjs, timezone: string): string => date.tz(timezone).format("'YYYY-MM-DD HH:mm:ss'")-const conversionMatch = (conversionGoal: WebAnalyticsConversionGoal | null): string | null => {if (conversionGoal && 'actionId' in conversionGoal) {return `matchesAction(${(conversionGoal as ActionConversionGoal).actionId})`@@ -252,55 +243,6 @@ const conversionMatch = (conversionGoal: WebAnalyticsConversionGoal | null): strreturn null}-const RAW_PATH = 'properties.$pathname'--export const buildPagePerformancePathExpr = (- source: string,- enabled: boolean,- filters: { regex?: string; alias?: string }[] | undefined-): string => {- if (!enabled || !filters?.length) {- return source- }- let expr = source- for (const f of filters) {- if (f.regex && f.alias != null) {- expr = `replaceRegexpAll(${expr}, ${escapeHogQLString(f.regex)}, ${escapeHogQLString(f.alias)})`- }- }- return expr-}--export const resolvePagePerformanceWindow = (- dateFilter: DateFilterState,- compareFilter: CompareFilter,- timezone: string,- now: dayjs.Dayjs = dayjs().tz(timezone)-): PagePerformanceWindow => {- const currentFrom =- dateFilter.dateFrom === 'all'- ? null- : (dateStringToDayJs(dateFilter.dateFrom, timezone) ?? now.subtract(7, 'day'))- const parsedCurrentTo = dateFilter.dateTo ? dateStringToDayJs(dateFilter.dateTo, timezone) : null- const currentTo =- parsedCurrentTo && /^\d{4}-\d{2}-\d{2}$/.test(dateFilter.dateTo ?? '')- ? parsedCurrentTo.endOf('day')- : (parsedCurrentTo ?? now)-- if (!compareFilter.compare || !currentFrom) {- return { currentFrom, currentTo, previousFrom: null, previousTo: null, timezone }- }-- const spanMs = Math.max(currentTo.diff(currentFrom), 1)- const compareToComponents = dateStringToComponents(compareFilter.compare_to ?? null)- const previousFrom = compareToComponents- ? componentsToDayJs(compareToComponents, currentFrom, timezone)- : currentFrom.subtract(spanMs, 'millisecond')- const previousTo = compareToComponents ? previousFrom.add(spanMs, 'millisecond') : currentFrom-- return { currentFrom, currentTo, previousFrom, previousTo, timezone }-}-export interface ParsedOverviewResponse {totals: Record<string, number>buckets: { bucket: dayjs.Dayjs; values: Record<string, number> }[]@@ -420,23 +362,6 @@ export const parseMetricCell = (value: unknown): MetricCellValue | null =>? { current: Number(value[0] ?? 0), previous: Number(value[1] ?? 0) }: null-/** Shares below 10% get a decimal, so a page holding 0.4% of site traffic doesn't read as 0%. */-export const formatShare = (part: number, whole: number): string | null => {- if (whole <= 0 || part <= 0) {- return null- }- const fraction = part / whole- return percentage(fraction, fraction < 0.1 ? 1 : 0)-}--/** Percent change against the previous period, or null when there is no comparable baseline. */-export const changeVsPrevious = (current: number, previous: number): number | null => {- if (previous === 0) {- return null- }- return current === previous ? 0 : current / previous - 1-}-const SORTABLE_COLUMNS = new Set(['breakdown_value','visitors',@@ -449,23 +374,6 @@ const SORTABLE_COLUMNS = new Set([const orderByExpr = (column: string): string =>`"context.columns.${SORTABLE_COLUMNS.has(column) ? column : 'visitors'}"`-const windowPredicates = (window: PagePerformanceWindow): { cur: string; prev: string; full: string } => {- const upperBound = `timestamp <= ${tsLiteral(window.currentTo, window.timezone)}`- const cur = window.currentFrom- ? `timestamp >= ${tsLiteral(window.currentFrom, window.timezone)} AND ${upperBound}`- : upperBound- const prev =- window.previousFrom && window.previousTo- ? `timestamp >= ${tsLiteral(window.previousFrom, window.timezone)} AND timestamp < ${tsLiteral(window.previousTo, window.timezone)}`- : 'false'- return { cur, prev, full: `(${cur}) OR (${prev})` }-}--const pageKeyExpr = (pathExpr: string): string => `concat(coalesce(properties.$host, ''), ${pathExpr})`--const pageCandidatesPredicate = (pageKey: string, candidates: string[]): string =>- candidates.length > 0 ? `${pageKey} IN (${candidates.map(escapeHogQLString).join(', ')})` : 'false'-/*** In the page_durations subquery, `{filters}` matches the carrier event (the NEXT pageview, or the* pageleave), not the measured page, so an event property that varies per pageview (e.g. $pathname)@@ -596,67 +504,6 @@ GROUP BY GROUPING SETS ((bucket), ())`}-const buildBreakdownQuery = (- metric: PagePerformanceMetric,- page: string,- window: PagePerformanceWindow,- pathExpr: string-): string | null => {- const { cur } = windowPredicates(window)- const pageMatch = `${pageKeyExpr(pathExpr)} = ${escapeHogQLString(page)}`-- if (metric === 'llm_referrals') {- return `-SELECT- coalesce(nullIf(session.$entry_referring_domain, ''), '(direct)') AS "Assistant",- uniq(person_id) AS "Visitors"-FROM events-WHERE and(${AI_VIEW}, ${pageMatch}, (${cur}), {filters})-GROUP BY "Assistant"-ORDER BY "Visitors" DESC-LIMIT 10-`- }- if (metric === 'agent_crawls') {- return `-SELECT- \`$virt_bot_name\` AS "Crawler",- \`$virt_traffic_category\` AS "Category",- count() AS "Requests",- max(timestamp) AS "Last seen"-FROM events-WHERE and(${CRAWLER}, event IN ${PAGE_PERFORMANCE_EVENTS}, ${pageMatch}, (${cur}), {filters})-GROUP BY "Crawler", "Category"-ORDER BY "Requests" DESC-LIMIT 50-`- }- return null-}--export const buildGoogleSearchConsoleQuery = (- tableName: string,- page: string,- window: PagePerformanceWindow,- isPathCleaningEnabled: boolean,- pathCleaningFilters: { regex?: string; alias?: string }[] | undefined-): string => {- const gscPath = buildPagePerformancePathExpr('path(page)', isPathCleaningEnabled, pathCleaningFilters)- const gscPageKey = `concat(domain(page), ${gscPath})`- const dateField = 'toDate(date)'- const dateUpperBound = `${dateField} <= toDate(${tsLiteral(window.currentTo, window.timezone)})`- const datePredicate = window.currentFrom- ? `${dateField} >= toDate(${tsLiteral(window.currentFrom, window.timezone)}) AND ${dateUpperBound}`- : dateUpperBound-- return `SELECT query AS "Query", sum(clicks) AS "Clicks", round(avg(position), 1) AS "Avg. position"-FROM ${escapeDottedHogQLIdentifier(tableName)}-WHERE ${gscPageKey} = ${escapeHogQLString(page)} AND (${datePredicate})-GROUP BY query-ORDER BY sum(clicks) DESC-LIMIT 20`-}-export interface pagePerformanceLogicValues {allActions: ActionType[]currentTeam: TeamPublicType | TeamType | nulldiff --git a/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsx b/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsxindex d9063defe76f..6deb719a6eaf 100644--- a/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsx+++ b/products/web_analytics/frontend/searchAndAi/SearchAndAiMetricCell.tsx@@ -8,13 +8,9 @@ import { IconTrendingDown, IconTrendingFlat } from 'lib/lemon-ui/icons'import { humanFriendlyDuration } from 'lib/utils/durations'import { percentage } from 'lib/utils/numbers'import { pluralize } from 'lib/utils/strings'-import {- changeVsPrevious,- formatShare,- PagePerformanceMetric,- pagePerformanceLogic,- parseMetricCell,-} from 'scenes/web-analytics/pagePerformanceLogic'+import { pagePerformanceLogic, parseMetricCell } from 'scenes/web-analytics/pagePerformanceLogic'++import { PagePerformanceMetric, changeVsPrevious, formatShare } from './searchAndAiQueries'export type SearchAndAiMetric =| 'visitors'diff --git a/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts b/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.tsnew file mode 100644index 000000000000..d2a06b28384a--- /dev/null+++ b/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts@@ -0,0 +1,177 @@+import { dayjs } from 'lib/dayjs'+import { componentsToDayJs, dateStringToComponents, dateStringToDayJs } from 'lib/utils/dateFilters'+import { percentage } from 'lib/utils/numbers'+import type { DateFilterState } from 'scenes/web-analytics/webAnalyticsLogic'++import { CompareFilter } from '~/queries/schema/schema-general'+import { escapeDottedHogQLIdentifier, escapeHogQLString } from '~/queries/utils'++export const PAGE_PERFORMANCE_EVENTS = "('$pageview', '$screen', '$http_log')"++export const HUMAN_EVENTS = "('$pageview', '$screen')"++export const HUMAN_VIEW = "(event = '$pageview' OR event = '$screen') AND NOT `$virt_is_bot`"++export const GOOGLE_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'Organic Search' AND session.$entry_referring_domain ILIKE '%google%'`++export const AI_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'AI'`++export const CRAWLER =+ "`$virt_is_bot` = true AND `$virt_bot_name` != '' AND `$virt_traffic_category` IN ('ai_crawler', 'ai_assistant', 'ai_search')"++export type PagePerformanceMetric = 'llm_referrals' | 'agent_crawls' | 'google_search'++export interface PagePerformanceWindow {+ currentFrom: dayjs.Dayjs | null+ currentTo: dayjs.Dayjs+ previousFrom: dayjs.Dayjs | null+ previousTo: dayjs.Dayjs | null+ timezone: string+}++export const tsLiteral = (date: dayjs.Dayjs, timezone: string): string =>+ date.tz(timezone).format("'YYYY-MM-DD HH:mm:ss'")++export const RAW_PATH = 'properties.$pathname'++export const buildPagePerformancePathExpr = (+ source: string,+ enabled: boolean,+ filters: { regex?: string; alias?: string }[] | undefined+): string => {+ if (!enabled || !filters?.length) {+ return source+ }+ let expr = source+ for (const f of filters) {+ if (f.regex && f.alias != null) {+ expr = `replaceRegexpAll(${expr}, ${escapeHogQLString(f.regex)}, ${escapeHogQLString(f.alias)})`+ }+ }+ return expr+}++export const resolvePagePerformanceWindow = (+ dateFilter: DateFilterState,+ compareFilter: CompareFilter,+ timezone: string,+ now: dayjs.Dayjs = dayjs().tz(timezone)+): PagePerformanceWindow => {+ const currentFrom =+ dateFilter.dateFrom === 'all'+ ? null+ : (dateStringToDayJs(dateFilter.dateFrom, timezone) ?? now.subtract(7, 'day'))+ const parsedCurrentTo = dateFilter.dateTo ? dateStringToDayJs(dateFilter.dateTo, timezone) : null+ const currentTo =+ parsedCurrentTo && /^\d{4}-\d{2}-\d{2}$/.test(dateFilter.dateTo ?? '')+ ? parsedCurrentTo.endOf('day')+ : (parsedCurrentTo ?? now)++ if (!compareFilter.compare || !currentFrom) {+ return { currentFrom, currentTo, previousFrom: null, previousTo: null, timezone }+ }++ const spanMs = Math.max(currentTo.diff(currentFrom), 1)+ const compareToComponents = dateStringToComponents(compareFilter.compare_to ?? null)+ const previousFrom = compareToComponents+ ? componentsToDayJs(compareToComponents, currentFrom, timezone)+ : currentFrom.subtract(spanMs, 'millisecond')+ const previousTo = compareToComponents ? previousFrom.add(spanMs, 'millisecond') : currentFrom++ return { currentFrom, currentTo, previousFrom, previousTo, timezone }+}++/** Shares below 10% get a decimal, so a page holding 0.4% of site traffic doesn't read as 0%. */+export const formatShare = (part: number, whole: number): string | null => {+ if (whole <= 0 || part <= 0) {+ return null+ }+ const fraction = part / whole+ return percentage(fraction, fraction < 0.1 ? 1 : 0)+}++/** Percent change against the previous period, or null when there is no comparable baseline. */+export const changeVsPrevious = (current: number, previous: number): number | null => {+ if (previous === 0) {+ return null+ }+ return current === previous ? 0 : current / previous - 1+}++export const windowPredicates = (window: PagePerformanceWindow): { cur: string; prev: string; full: string } => {+ const upperBound = `timestamp <= ${tsLiteral(window.currentTo, window.timezone)}`+ const cur = window.currentFrom+ ? `timestamp >= ${tsLiteral(window.currentFrom, window.timezone)} AND ${upperBound}`+ : upperBound+ const prev =+ window.previousFrom && window.previousTo+ ? `timestamp >= ${tsLiteral(window.previousFrom, window.timezone)} AND timestamp < ${tsLiteral(window.previousTo, window.timezone)}`+ : 'false'+ return { cur, prev, full: `(${cur}) OR (${prev})` }+}++export const pageKeyExpr = (pathExpr: string): string => `concat(coalesce(properties.$host, ''), ${pathExpr})`++export const pageCandidatesPredicate = (pageKey: string, candidates: string[]): string =>+ candidates.length > 0 ? `${pageKey} IN (${candidates.map(escapeHogQLString).join(', ')})` : 'false'++export const buildBreakdownQuery = (+ metric: PagePerformanceMetric,+ page: string,+ window: PagePerformanceWindow,+ pathExpr: string+): string | null => {+ const { cur } = windowPredicates(window)+ const pageMatch = `${pageKeyExpr(pathExpr)} = ${escapeHogQLString(page)}`++ if (metric === 'llm_referrals') {+ return `+SELECT+ coalesce(nullIf(session.$entry_referring_domain, ''), '(direct)') AS "Assistant",+ uniq(person_id) AS "Visitors"+FROM events+WHERE and(${AI_VIEW}, ${pageMatch}, (${cur}), {filters})+GROUP BY "Assistant"+ORDER BY "Visitors" DESC+LIMIT 10+`+ }+ if (metric === 'agent_crawls') {+ return `+SELECT+ \`$virt_bot_name\` AS "Crawler",+ \`$virt_traffic_category\` AS "Category",+ count() AS "Requests",+ max(timestamp) AS "Last seen"+FROM events+WHERE and(${CRAWLER}, event IN ${PAGE_PERFORMANCE_EVENTS}, ${pageMatch}, (${cur}), {filters})+GROUP BY "Crawler", "Category"+ORDER BY "Requests" DESC+LIMIT 50+`+ }+ return null+}++export const buildGoogleSearchConsoleQuery = (+ tableName: string,+ page: string,+ window: PagePerformanceWindow,+ isPathCleaningEnabled: boolean,+ pathCleaningFilters: { regex?: string; alias?: string }[] | undefined+): string => {+ const gscPath = buildPagePerformancePathExpr('path(page)', isPathCleaningEnabled, pathCleaningFilters)+ const gscPageKey = `concat(domain(page), ${gscPath})`+ const dateField = 'toDate(date)'+ const dateUpperBound = `${dateField} <= toDate(${tsLiteral(window.currentTo, window.timezone)})`+ const datePredicate = window.currentFrom+ ? `${dateField} >= toDate(${tsLiteral(window.currentFrom, window.timezone)}) AND ${dateUpperBound}`+ : dateUpperBound++ return `SELECT query AS "Query", sum(clicks) AS "Clicks", round(avg(position), 1) AS "Avg. position"+FROM ${escapeDottedHogQLIdentifier(tableName)}+WHERE ${gscPageKey} = ${escapeHogQLString(page)} AND (${datePredicate})+GROUP BY query+ORDER BY sum(clicks) DESC+LIMIT 20`+}