Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(web-analytics): share search and ai cards with page reports (#123505)

Jordan Mryyan5d53846eedeb
diff --git a/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx b/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx
index 11868c1021d0..139387ea07f5 100644
--- a/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx
+++ b/frontend/src/scenes/web-analytics/PagePerformanceBreakdownModal.tsx
@@ -1,27 +1,14 @@
import { useActions, useValues } from 'kea'
-import { LemonBanner, LemonButton, LemonModal } from '@posthog/lemon-ui'
+import { LemonBanner, LemonModal } from '@posthog/lemon-ui'
-import { useOnMountEffect } from 'lib/hooks/useOnMountEffect'
-import { IconOpenInNew } from 'lib/lemon-ui/icons'
-import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'
-
-import { DataTableNode, NodeKind } from '~/queries/schema/schema-general'
-import { InsightLogicProps } from '~/types'
-
-import { SearchAndAiLoading } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiLoading'
-import {
- PagePerformanceMetric,
- buildGoogleSearchConsoleQuery,
-} from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'
+import { GoogleSearchConsoleKeywords } from 'products/web_analytics/frontend/searchAndAi/GoogleSearchConsoleKeywords'
+import { PagePerformanceMetric } from 'products/web_analytics/frontend/searchAndAi/searchAndAiQueries'
import { SearchAndAiQuery } from 'products/web_analytics/frontend/searchAndAi/SearchAndAiQuery'
-import { TileId, WEB_ANALYTICS_DEFAULT_QUERY_TAGS } from './common'
+import { TileId } from './common'
import { createPagePerformanceInsightProps, pagePerformanceLogic } from './pagePerformanceLogic'
-const GSC_DOCS_URL = 'https://posthog.com/docs/cdp/sources/google-search-console'
-const GSC_TABLE_NAME = 'search_analytics_by_query_page'
-
const MODAL_TITLE: Record<PagePerformanceMetric, string> = {
llm_referrals: 'AI referrals',
agent_crawls: 'AI crawls',
@@ -34,71 +21,9 @@ const MODAL_SUBTITLE: Record<PagePerformanceMetric, string> = {
google_search: 'Search queries this page ranks for, from Google Search Console.',
}
-const GoogleBreakdown = ({ page, insightProps }: { page: string; insightProps: InsightLogicProps }): JSX.Element => {
- const { dataWarehouseTables, database, databaseLoading, databaseLoadError } = useValues(databaseTableListLogic)
- const { loadDatabase } = useActions(databaseTableListLogic)
- const { currentTeam, isPathCleaningEnabled, window } = useValues(pagePerformanceLogic)
-
- useOnMountEffect(() => {
- if (!database) {
- loadDatabase()
- }
- })
-
- const gscTable = dataWarehouseTables.find((t) => t.name.includes(GSC_TABLE_NAME))
-
- if (!gscTable) {
- if (databaseLoadError && !databaseLoading) {
- return (
- <LemonBanner
- type="error"
- action={{
- children: 'Try again',
- onClick: () => loadDatabase({ force: true }),
- loading: databaseLoading,
- }}
- >
- Could not load data warehouse tables. Try again to check for your Google Search Console source.
- </LemonBanner>
- )
- }
- if (databaseLoading || !database) {
- return <SearchAndAiLoading loading label="Checking Google Search Console" className="min-h-64" />
- }
- return (
- <div className="flex flex-col items-center gap-3 py-8 text-center">
- <p className="text-secondary max-w-md m-0">
- Google Search Console shows the search queries and average positions each page ranks for. Connect it
- as a data warehouse source to unlock this breakdown.
- </p>
- <LemonButton type="primary" to={GSC_DOCS_URL} targetBlank sideIcon={<IconOpenInNew />}>
- Connect Google Search Console
- </LemonButton>
- </div>
- )
- }
-
- const query: DataTableNode = {
- kind: NodeKind.DataTableNode,
- source: {
- kind: NodeKind.HogQLQuery,
- query: buildGoogleSearchConsoleQuery(
- gscTable.name,
- page,
- window,
- isPathCleaningEnabled,
- currentTeam?.path_cleaning_filters
- ),
- tags: WEB_ANALYTICS_DEFAULT_QUERY_TAGS,
- },
- embedded: true,
- showActions: false,
- }
- return <SearchAndAiQuery uniqueKey="page-performance-gsc-breakdown" query={query} insightProps={insightProps} />
-}
-
export const PagePerformanceBreakdownModal = (): JSX.Element => {
- const { breakdownModal, breakdownQuery } = useValues(pagePerformanceLogic)
+ const { breakdownModal, breakdownQuery, currentTeam, isPathCleaningEnabled, window } =
+ useValues(pagePerformanceLogic)
const { closeBreakdown } = useActions(pagePerformanceLogic)
const isOpen = !!breakdownModal
@@ -108,7 +33,17 @@ export const PagePerformanceBreakdownModal = (): JSX.Element => {
let body: JSX.Element | null = null
if (metric === 'google_search' && isOpen) {
- body = <GoogleBreakdown page={page} insightProps={insightProps} />
+ body = (
+ <GoogleSearchConsoleKeywords
+ page={page}
+ window={window}
+ isPathCleaningEnabled={isPathCleaningEnabled}
+ pathCleaningFilters={currentTeam?.path_cleaning_filters}
+ insightProps={insightProps}
+ uniqueKey="page-performance-gsc-breakdown"
+ connectPrompt="text"
+ />
+ )
} else if (breakdownQuery) {
body = (
<SearchAndAiQuery
diff --git a/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts b/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts
index 6e72495c0446..491e67198318 100644
--- a/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts
+++ b/frontend/src/scenes/web-analytics/pagePerformanceLogic.ts
@@ -28,8 +28,6 @@ import {
CompareFilter,
CustomEventConversionGoal,
DataTableNode,
- EventsNode,
- GroupNode,
HogQLQuery,
InsightVizNode,
NodeKind,
@@ -42,16 +40,7 @@ import {
WebStatsTableQuery,
} from '~/queries/schema/schema-general'
import { escapeHogQLString } from '~/queries/utils'
-import {
- ActionType,
- BaseMathType,
- ChartDisplayType,
- FilterLogicalOperator,
- InsightLogicProps,
- PropertyFilterType,
- PropertyOperator,
- TeamType,
-} from '~/types'
+import { ActionType, BaseMathType, ChartDisplayType, InsightLogicProps, TeamType } from '~/types'
import {
AI_VIEW,
@@ -63,6 +52,9 @@ import {
PagePerformanceMetric,
PagePerformanceWindow,
RAW_PATH,
+ AI_CHANNEL_FILTER,
+ AI_CRAWLER_FILTERS,
+ aiCrawlerRequestSeries,
buildBreakdownQuery,
buildPagePerformancePathExpr,
pageCandidatesPredicate,
@@ -73,7 +65,6 @@ import {
import type { TeamPublicType } from '../../types'
import {
- BOT_ANALYTICS_EVENTS,
TileId,
WEB_ANALYTICS_DATA_COLLECTION_NODE_ID,
WEB_ANALYTICS_DEFAULT_QUERY_TAGS,
@@ -870,36 +861,8 @@ export const pagePerformanceLogic: LogicWrapper<pagePerformanceLogicType> = kea<
const dateRange = { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }
const interval = dateFilter.interval
- const referralFilters: WebAnalyticsPropertyFilters = [
- {
- key: '$channel_type',
- value: ['AI'],
- operator: PropertyOperator.Exact,
- type: PropertyFilterType.Session,
- },
- ...webAnalyticsFilters,
- ]
- const crawlerFilters: WebAnalyticsPropertyFilters = [
- {
- key: '$virt_is_bot',
- value: ['true'],
- operator: PropertyOperator.Exact,
- type: PropertyFilterType.Event,
- },
- {
- key: '$virt_bot_name',
- value: [''],
- operator: PropertyOperator.IsNot,
- type: PropertyFilterType.Event,
- },
- {
- key: '$virt_traffic_category',
- value: ['ai_crawler', 'ai_assistant', 'ai_search'],
- operator: PropertyOperator.Exact,
- type: PropertyFilterType.Event,
- },
- ...webAnalyticsFilters,
- ]
+ const referralFilters: WebAnalyticsPropertyFilters = [AI_CHANNEL_FILTER, ...webAnalyticsFilters]
+ const crawlerFilters: WebAnalyticsPropertyFilters = [...AI_CRAWLER_FILTERS, ...webAnalyticsFilters]
const referralColumns = [
'breakdown_value',
@@ -944,24 +907,6 @@ export const pagePerformanceLogic: LogicWrapper<pagePerformanceLogicType> = kea<
embedded: false,
})
- const crawlerRequestSeries: GroupNode[] = [
- {
- kind: NodeKind.GroupNode,
- name: BOT_ANALYTICS_EVENTS.join(', '),
- custom_name: 'Requests',
- operator: FilterLogicalOperator.Or,
- nodes: BOT_ANALYTICS_EVENTS.map(
- (event): EventsNode => ({
- event,
- kind: NodeKind.EventsNode as const,
- math: BaseMathType.TotalCount,
- name: event,
- })
- ),
- math: BaseMathType.TotalCount,
- },
- ]
-
return {
referralTrend: {
kind: NodeKind.InsightVizNode,
@@ -997,7 +942,7 @@ export const pagePerformanceLogic: LogicWrapper<pagePerformanceLogicType> = kea<
kind: NodeKind.TrendsQuery,
dateRange,
interval,
- series: crawlerRequestSeries,
+ series: [aiCrawlerRequestSeries('Requests')],
trendsFilter: { display: ChartDisplayType.ActionsLineGraph } as TrendsFilter,
properties: crawlerFilters,
filterTestAccounts,
diff --git a/products/web_analytics/frontend/searchAndAi/AiAssistantList.tsx b/products/web_analytics/frontend/searchAndAi/AiAssistantList.tsx
new file mode 100644
index 000000000000..5ee7f809a0d5
--- /dev/null
+++ b/products/web_analytics/frontend/searchAndAi/AiAssistantList.tsx
@@ -0,0 +1,77 @@
+import { LemonButton, LemonSkeleton } from '@posthog/lemon-ui'
+
+import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
+import { percentage } from 'lib/utils/numbers'
+import { pluralize } from 'lib/utils/strings'
+
+import { AiBrandLogo } from './AiBrandLogo'
+
+export interface AiAssistantRow {
+ assistant: string
+ visitors: number
+ views: number
+}
+
+export interface AiAssistantListProps {
+ rows: AiAssistantRow[] | null
+ loading: boolean
+ emptyText: string
+ failed?: boolean
+ onRetry?: () => void
+}
+
+export function AiAssistantList({ rows, loading, emptyText, failed, onRetry }: AiAssistantListProps): JSX.Element {
+ if (failed && !loading) {
+ return (
+ <div className="flex flex-col items-center gap-2 p-6 text-center">
+ <p className="m-0 text-secondary">Couldn't load visitors by assistant.</p>
+ {onRetry && (
+ <LemonButton type="secondary" size="small" onClick={onRetry}>
+ Try again
+ </LemonButton>
+ )}
+ </div>
+ )
+ }
+ if (loading && !rows) {
+ return (
+ <div className="flex flex-col gap-3 p-4">
+ {[0, 1, 2].map((index) => (
+ <LemonSkeleton key={index} className="h-6 w-full" />
+ ))}
+ </div>
+ )
+ }
+ if (!rows || rows.length === 0) {
+ return <p className="m-0 p-6 text-center text-secondary">{emptyText}</p>
+ }
+ const max = Math.max(...rows.map((row) => row.visitors), 1)
+ const total = rows.reduce((sum, row) => sum + row.visitors, 0)
+ return (
+ <ol className="@container/assistant-list m-0 p-0 list-none flex flex-col" data-attr="ai-assistant-list">
+ {rows.map((row) => {
+ const share = total > 0 ? row.visitors / total : 0
+ return (
+ <li
+ key={row.assistant}
+ className="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-x-3 gap-y-1.5 px-4 py-2.5 border-t first:border-t-0 @min-[30rem]/assistant-list:grid-cols-[minmax(0,10rem)_minmax(0,1fr)_7rem]"
+ >
+ <span className="flex items-center gap-2 min-w-0">
+ <AiBrandLogo name={row.assistant} />
+ <span className="font-medium truncate">{row.assistant}</span>
+ </span>
+ <span className="col-span-2 order-last @min-[30rem]/assistant-list:col-span-1 @min-[30rem]/assistant-list:order-none">
+ <LemonProgress percent={(row.visitors / max) * 100} strokeColor="var(--data-color-3)" />
+ </span>
+ <span className="flex flex-col items-end leading-tight tabular-nums" translate="no">
+ <span className="font-semibold">{row.visitors.toLocaleString()}</span>
+ <span className="text-xs text-secondary">
+ {percentage(share, 0)} · {pluralize(row.views, 'view')}
+ </span>
+ </span>
+ </li>
+ )
+ })}
+ </ol>
+ )
+}
diff --git a/products/web_analytics/frontend/searchAndAi/AiTableCard.tsx b/products/web_analytics/frontend/searchAndAi/AiTableCard.tsx
new file mode 100644
index 000000000000..e7559c09e821
--- /dev/null
+++ b/products/web_analytics/frontend/searchAndAi/AiTableCard.tsx
@@ -0,0 +1,53 @@
+import { useMemo } from 'react'
+
+import { TileId } from 'scenes/web-analytics/common'
+import { PagePerformanceCard } from 'scenes/web-analytics/PagePerformanceCard'
+import { PagePerformanceCardHeader } from 'scenes/web-analytics/PagePerformanceCardHeader'
+import { createPagePerformanceInsightProps } from 'scenes/web-analytics/pagePerformanceLogic'
+import { webAnalyticsDataTableQueryContext } from 'scenes/web-analytics/tiles/WebAnalyticsTile'
+
+import { DataTableNode } from '~/queries/schema/schema-general'
+import { QueryContext } from '~/queries/types'
+
+import { SearchAndAiQuery } from './SearchAndAiQuery'
+
+export function AiTableCard({
+ title,
+ query,
+ tileId,
+ columns,
+ scope = 'table',
+ tableLayout = 'fixed',
+ footer,
+}: {
+ title: string
+ query: DataTableNode
+ tileId: TileId
+ columns?: QueryContext['columns']
+ scope?: string
+ tableLayout?: 'auto' | 'fixed'
+ footer?: React.ReactNode
+}): JSX.Element {
+ const context = useMemo(
+ (): QueryContext => ({
+ ...webAnalyticsDataTableQueryContext,
+ ...(columns ? { columns: { ...webAnalyticsDataTableQueryContext.columns, ...columns } } : {}),
+ insightProps: createPagePerformanceInsightProps(tileId, scope),
+ showLoadNextButton: true,
+ tableLayout,
+ suppressSlowQuerySuggestions: true,
+ }),
+ [tileId, columns, scope, tableLayout]
+ )
+ return (
+ <PagePerformanceCard className="flex-1 min-w-0" footer={footer}>
+ <SearchAndAiQuery
+ uniqueKey={scope === 'table' ? `page-performance-${tileId}` : `page-performance-${tileId}-${scope}`}
+ query={query}
+ insightProps={context.insightProps!}
+ context={context}
+ header={<PagePerformanceCardHeader title={title} />}
+ />
+ </PagePerformanceCard>
+ )
+}
diff --git a/products/web_analytics/frontend/searchAndAi/AiTrendCard.tsx b/products/web_analytics/frontend/searchAndAi/AiTrendCard.tsx
new file mode 100644
index 000000000000..adfb61261f0e
--- /dev/null
+++ b/products/web_analytics/frontend/searchAndAi/AiTrendCard.tsx
@@ -0,0 +1,51 @@
+import clsx from 'clsx'
+import { useMountedLogic } from 'kea'
+import { useMemo } from 'react'
+
+import { TileId } from 'scenes/web-analytics/common'
+import { createPagePerformanceInsightProps } from 'scenes/web-analytics/pagePerformanceLogic'
+import { WebQuery } from 'scenes/web-analytics/tiles/WebAnalyticsTile'
+import { webAnalyticsLogic } from 'scenes/web-analytics/webAnalyticsLogic'
+
+import { InsightVizNode } from '~/queries/schema/schema-general'
+
+import { SearchAndAiQuery } from './SearchAndAiQuery'
+import { SearchAndAiTrendHeader } from './SearchAndAiTrendHeader'
+
+export function AiTrendCard({
+ title,
+ query,
+ tileId,
+ uniqueKey,
+ scope,
+ className,
+}: {
+ title: string
+ query: InsightVizNode
+ tileId: TileId
+ uniqueKey: string
+ scope?: string
+ className?: string
+}): JSX.Element {
+ const logic = useMountedLogic(webAnalyticsLogic)
+ const insightProps = useMemo(() => createPagePerformanceInsightProps(tileId, scope), [tileId, scope])
+ return (
+ <div className={clsx('min-h-88 min-w-0 flex flex-col', className)}>
+ <SearchAndAiQuery
+ uniqueKey={uniqueKey}
+ query={query}
+ insightProps={insightProps}
+ renderQuery={(insightProps) => (
+ <WebQuery
+ attachTo={logic}
+ uniqueKey={uniqueKey}
+ query={query}
+ insightProps={insightProps}
+ tileId={tileId}
+ headerSlot={<SearchAndAiTrendHeader title={title} />}
+ />
+ )}
+ />
+ </div>
+ )
+}
diff --git a/products/web_analytics/frontend/searchAndAi/GoogleSearchConsoleKeywords.tsx b/products/web_analytics/frontend/searchAndAi/GoogleSearchConsoleKeywords.tsx
new file mode 100644
index 000000000000..660d020c2ce5
--- /dev/null
+++ b/products/web_analytics/frontend/searchAndAi/GoogleSearchConsoleKeywords.tsx
@@ -0,0 +1,145 @@
+import { useActions, useValues } from 'kea'
+import { useMemo } from 'react'
+
+import * as magnifyingGlassPng from '@posthog/brand/hoggies/png/magnifying-glass'
+import { LemonBanner, LemonButton } from '@posthog/lemon-ui'
+
+import { pngHoggie } from 'lib/brand/hoggies'
+import { useOnMountEffect } from 'lib/hooks/useOnMountEffect'
+import { IconOpenInNew } from 'lib/lemon-ui/icons'
+import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'
+import { WEB_ANALYTICS_DEFAULT_QUERY_TAGS } from 'scenes/web-analytics/common'
+
+import { DataTableNode, NodeKind } from '~/queries/schema/schema-general'
+import { InsightLogicProps, PathCleaningFilter } from '~/types'
+
+import { SearchAndAiLoading } from './SearchAndAiLoading'
+import {
+ GOOGLE_SEARCH_CONSOLE_DOCS_URL,
+ PagePerformanceWindow,
+ buildGoogleSearchConsoleQuery,
+ findGoogleSearchConsoleTable,
+} from './searchAndAiQueries'
+import { SearchAndAiQuery } from './SearchAndAiQuery'
+
+const SearchHog = pngHoggie(magnifyingGlassPng)
+
+export interface GoogleSearchConsoleKeywordsProps {
+ page: string
+ window: PagePerformanceWindow
+ isPathCleaningEnabled: boolean
+ pathCleaningFilters: PathCleaningFilter[] | undefined
+ insightProps: InsightLogicProps
+ uniqueKey: string
+ connectPrompt?: 'illustration' | 'text'
+ onConnectClick?: () => void
+}
+
+export function GoogleSearchConsoleKeywords({
+ page,
+ window,
+ isPathCleaningEnabled,
+ pathCleaningFilters,
+ insightProps,
+ uniqueKey,
+ connectPrompt = 'illustration',
+ onConnectClick,
+}: GoogleSearchConsoleKeywordsProps): JSX.Element {
+ const { dataWarehouseTables, database, databaseLoading, databaseLoadError } = useValues(databaseTableListLogic)
+ const { loadDatabase } = useActions(databaseTableListLogic)
+
+ useOnMountEffect(() => {
+ if (!database) {
+ loadDatabase()
+ }
+ })
+
+ const gscTableName = findGoogleSearchConsoleTable(dataWarehouseTables)
+ const query = useMemo(
+ (): DataTableNode | null =>
+ gscTableName
+ ? {
+ kind: NodeKind.DataTableNode,
+ source: {
+ kind: NodeKind.HogQLQuery,
+ query: buildGoogleSearchConsoleQuery(
+ gscTableName,
+ page,
+ window,
+ isPathCleaningEnabled,
+ pathCleaningFilters
+ ),
+ tags: WEB_ANALYTICS_DEFAULT_QUERY_TAGS,
+ },
+ embedded: true,
+ showActions: false,
+ }
+ : null,
+ [gscTableName, page, window, isPathCleaningEnabled, pathCleaningFilters]
+ )
+
+ if (!query) {
+ if (databaseLoadError && !databaseLoading) {
+ return (
+ <LemonBanner
+ type="error"
+ action={{
+ children: 'Try again',
+ onClick: () => loadDatabase({ force: true }),
+ loading: databaseLoading,
+ }}
+ >
+ Could not load data warehouse tables. Try again to check for your Google Search Console source.
+ </LemonBanner>
+ )
+ }
+ if (databaseLoading || !database) {
+ return <SearchAndAiLoading loading label="Checking Google Search Console" className="min-h-64" />
+ }
+ if (connectPrompt === 'text') {
+ return (
+ <div className="flex flex-col items-center gap-3 py-8 text-center">
+ <p className="text-secondary max-w-md m-0">
+ Google Search Console shows the search queries and average positions each page ranks for.
+ Connect it as a data warehouse source to unlock this breakdown.
+ </p>
+ <LemonButton
+ type="primary"
+ to={GOOGLE_SEARCH_CONSOLE_DOCS_URL}
+ targetBlank
+ sideIcon={<IconOpenInNew />}
+ >
+ Connect Google Search Console
+ </LemonButton>
+ </div>
+ )
+ }
+ return (
+ <div
+ className="flex flex-col items-center gap-4 px-6 py-8 text-center @container"
+ data-attr="gsc-connect-empty-state"
+ >
+ <SearchHog title="" className="w-28" />
+ <div className="flex flex-col gap-1 max-w-md">
+ <h3 className="m-0 text-base font-semibold">See what people search to find this page</h3>
+ <p className="m-0 text-secondary">
+ Connect Google Search Console as a data warehouse source to see the queries this page ranks for,
+ its clicks and its average position.
+ </p>
+ </div>
+ <LemonButton
+ type="primary"
+ to={GOOGLE_SEARCH_CONSOLE_DOCS_URL}
+ targetBlank
+ sideIcon={<IconOpenInNew />}
+ onClick={onConnectClick}
+ data-attr="page-report-gsc-connect"
+ >
+ Connect Google Search Console
+ </LemonButton>
+ </div>
+ )
+ }
+
+ return <SearchAndAiQuery uniqueKey={uniqueKey} query={query} insightProps={insightProps} />
+}
diff --git a/products/web_analytics/frontend/searchAndAi/SearchAndAiSectionHeading.tsx b/products/web_analytics/frontend/searchAndAi/SearchAndAiSectionHeading.tsx
new file mode 100644
index 000000000000..096fcebe4819
--- /dev/null
+++ b/products/web_analytics/frontend/searchAndAi/SearchAndAiSectionHeading.tsx
@@ -0,0 +1,14 @@
+export function SearchAndAiSectionHeading({
+ children,
+ description,
+}: {
+ children: React.ReactNode
+ description?: string
+}): JSX.Element {
+ return (
+ <div className="mb-4">
+ <h2 className="mb-0 text-xl font-semibold text-primary">{children}</h2>
+ {description ? <p className="m-0 text-sm text-secondary">{description}</p> : null}
+ </div>
+ )
+}
diff --git a/products/web_analytics/frontend/searchAndAi/crawlerColumns.tsx b/products/web_analytics/frontend/searchAndAi/crawlerColumns.tsx
new file mode 100644
index 000000000000..33adcec14b42
--- /dev/null
+++ b/products/web_analytics/frontend/searchAndAi/crawlerColumns.tsx
@@ -0,0 +1,52 @@
+import { TZLabel } from 'lib/components/TZLabel'
+import { CATEGORY_LABELS } from 'lib/utils/botDetection'
+import { humanFriendlyNumber } from 'lib/utils/numbers'
+
+import { QueryContextColumn } from '~/queries/types'
+
+import { AiBrandLogo } from './AiBrandLogo'
+
+const categoryColumn: QueryContextColumn = {
+ title: 'Category',
+ render: ({ value }) => (
+ <span>{(CATEGORY_LABELS as Record<string, string>)[String(value ?? '')] ?? String(value ?? '')}</span>
+ ),
+}
+
+const requestsColumn: QueryContextColumn = {
+ title: 'Requests',
+ align: 'right',
+ render: ({ value }) => (
+ <span className="tabular-nums" translate="no">
+ {humanFriendlyNumber(Number(value ?? 0))}
+ </span>
+ ),
+}
+
+const crawlersColumn: QueryContextColumn = { ...requestsColumn, title: 'Crawlers' }
+
+const crawlerColumn: QueryContextColumn = {
+ title: 'Crawler',
+ render: ({ value }) => {
+ const name = String(value ?? '')
+ return (
+ <span className="flex items-center gap-2 min-w-0">
+ <AiBrandLogo name={name} size="sm" />
+ <span className="truncate">{name}</span>
+ </span>
+ )
+ },
+}
+
+const lastSeenColumn: QueryContextColumn = {
+ title: 'Last seen',
+ render: ({ value }) => (value ? <TZLabel time={String(value)} /> : <span className="text-secondary">-</span>),
+}
+
+export const CRAWLER_COLUMNS: Record<string, QueryContextColumn> = {
+ Crawler: crawlerColumn,
+ Category: categoryColumn,
+ Requests: requestsColumn,
+ Crawlers: crawlersColumn,
+ 'Last seen': lastSeenColumn,
+}
diff --git a/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts b/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts
index d2a06b28384a..b50b04b4c8c1 100644
--- a/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts
+++ b/products/web_analytics/frontend/searchAndAi/searchAndAiQueries.ts
@@ -1,10 +1,23 @@
import { dayjs } from 'lib/dayjs'
import { componentsToDayJs, dateStringToComponents, dateStringToDayJs } from 'lib/utils/dateFilters'
import { percentage } from 'lib/utils/numbers'
+import { BOT_ANALYTICS_EVENTS, WEB_ANALYTICS_DEFAULT_QUERY_TAGS } from 'scenes/web-analytics/common'
import type { DateFilterState } from 'scenes/web-analytics/webAnalyticsLogic'
-import { CompareFilter } from '~/queries/schema/schema-general'
+import { performQuery } from '~/queries/query'
+import {
+ CompareFilter,
+ EventsNode,
+ GroupNode,
+ HogQLQuery,
+ NodeKind,
+ RefreshType,
+ WebAnalyticsPropertyFilters,
+} from '~/queries/schema/schema-general'
import { escapeDottedHogQLIdentifier, escapeHogQLString } from '~/queries/utils'
+import { BaseMathType, FilterLogicalOperator, PropertyFilterType, PropertyOperator } from '~/types'
+
+import type { AiAssistantRow } from './AiAssistantList'
export const PAGE_PERFORMANCE_EVENTS = "('$pageview', '$screen', '$http_log')"
@@ -19,6 +32,40 @@ export const AI_VIEW = `${HUMAN_VIEW} AND session.$channel_type = 'AI'`
export const CRAWLER =
"`$virt_is_bot` = true AND `$virt_bot_name` != '' AND `$virt_traffic_category` IN ('ai_crawler', 'ai_assistant', 'ai_search')"
+export const AI_CHANNEL_FILTER: WebAnalyticsPropertyFilters[number] = {
+ key: '$channel_type',
+ value: ['AI'],
+ operator: PropertyOperator.Exact,
+ type: PropertyFilterType.Session,
+}
+
+export const AI_CRAWLER_FILTERS: WebAnalyticsPropertyFilters = [
+ { key: '$virt_is_bot', value: ['true'], operator: PropertyOperator.Exact, type: PropertyFilterType.Event },
+ { key: '$virt_bot_name', value: [''], operator: PropertyOperator.IsNot, type: PropertyFilterType.Event },
+ {
+ key: '$virt_traffic_category',
+ value: ['ai_crawler', 'ai_assistant', 'ai_search'],
+ operator: PropertyOperator.Exact,
+ type: PropertyFilterType.Event,
+ },
+]
+
+export const aiCrawlerRequestSeries = (customName: string): GroupNode => ({
+ kind: NodeKind.GroupNode,
+ name: BOT_ANALYTICS_EVENTS.join(', '),
+ custom_name: customName,
+ operator: FilterLogicalOperator.Or,
+ nodes: BOT_ANALYTICS_EVENTS.map(
+ (event): EventsNode => ({
+ event,
+ kind: NodeKind.EventsNode,
+ math: BaseMathType.TotalCount,
+ name: event,
+ })
+ ),
+ math: BaseMathType.TotalCount,
+})
+
export type PagePerformanceMetric = 'llm_referrals' | 'agent_crawls' | 'google_search'
export interface PagePerformanceWindow {
@@ -29,8 +76,7 @@ export interface PagePerformanceWindow {
timezone: string
}
-export const tsLiteral = (date: dayjs.Dayjs, timezone: string): string =>
- date.tz(timezone).format("'YYYY-MM-DD HH:mm:ss'")
+const tsLiteral = (date: dayjs.Dayjs, timezone: string): string => date.tz(timezone).format("'YYYY-MM-DD HH:mm:ss'")
export const RAW_PATH = 'properties.$pathname'
@@ -55,7 +101,7 @@ export const resolvePagePerformanceWindow = (
dateFilter: DateFilterState,
compareFilter: CompareFilter,
timezone: string,
- now: dayjs.Dayjs = dayjs().tz(timezone)
+ now: dayjs.Dayjs = dayjs().tz(timezone).startOf('minute')
): PagePerformanceWindow => {
const currentFrom =
dateFilter.dateFrom === 'all'
@@ -153,25 +199,136 @@ LIMIT 50
return null
}
-export const buildGoogleSearchConsoleQuery = (
- tableName: string,
+const gscPageAndDatePredicate = (
page: string,
window: PagePerformanceWindow,
isPathCleaningEnabled: boolean,
pathCleaningFilters: { regex?: string; alias?: string }[] | undefined
): string => {
const gscPath = buildPagePerformancePathExpr('path(page)', isPathCleaningEnabled, pathCleaningFilters)
- const gscPageKey = `concat(domain(page), ${gscPath})`
const dateField = 'toDate(date)'
const dateUpperBound = `${dateField} <= toDate(${tsLiteral(window.currentTo, window.timezone)})`
const datePredicate = window.currentFrom
? `${dateField} >= toDate(${tsLiteral(window.currentFrom, window.timezone)}) AND ${dateUpperBound}`
: dateUpperBound
+ return `concat(domain(page), ${gscPath}) = ${escapeHogQLString(page)} AND (${datePredicate})`
+}
- return `SELECT query AS "Query", sum(clicks) AS "Clicks", round(avg(position), 1) AS "Avg. position"
+export const buildGoogleSearchConsoleQuery = (
+ tableName: string,
+ page: string,
+ window: PagePerformanceWindow,
+ isPathCleaningEnabled: boolean,
+ pathCleaningFilters: { regex?: string; alias?: string }[] | undefined
+): string =>
+ `SELECT query AS "Query", sum(clicks) AS "Clicks", round(avg(position), 1) AS "Avg. position"
FROM ${escapeDottedHogQLIdentifier(tableName)}
-WHERE ${gscPageKey} = ${escapeHogQLString(page)} AND (${datePredicate})
+WHERE ${gscPageAndDatePredicate(page, window, isPathCleaningEnabled, pathCleaningFilters)}
GROUP BY query
ORDER BY sum(clicks) DESC
LIMIT 20`
+
+const AI_ASSISTANT_NAMES: [string[], string][] = [
+ [['chatgpt', 'openai'], 'ChatGPT'],
+ [['claude'], 'Claude'],
+ [['perplexity'], 'Perplexity'],
+ [['gemini', 'bard.google'], 'Gemini'],
+ [['copilot'], 'Copilot'],
+ [['grok'], 'Grok'],
+ [['meta.ai'], 'Meta AI'],
+ [['deepseek'], 'DeepSeek'],
+ [['mistral'], 'Mistral'],
+ [['kagi'], 'Kagi'],
+ [['you.com'], 'You.com'],
+ [['phind'], 'Phind'],
+]
+
+const AI_ASSISTANT_SOURCE =
+ "if(lookupOrganicSourceType(session.$entry_utm_source) = 'AI', session.$entry_utm_source, session.$entry_referring_domain)"
+
+const aiAssistantBranch = ([terms, name]: [string[], string]): string => {
+ const condition = terms
+ .map((term) => `positionCaseInsensitive(${AI_ASSISTANT_SOURCE}, ${escapeHogQLString(term)}) > 0`)
+ .join(' OR ')
+ return `${condition}, ${escapeHogQLString(name)}`
+}
+
+const AI_ASSISTANT_EXPR = `multiIf(${AI_ASSISTANT_NAMES.map(aiAssistantBranch).join(', ')}, coalesce(nullIf(nullIf(${AI_ASSISTANT_SOURCE}, ''), '$direct'), 'Unattributed'))`
+
+const aiSessionsPredicate = (window: PagePerformanceWindow): string => {
+ const lowerBound = window.currentFrom
+ ? `$start_timestamp >= ${tsLiteral(window.currentFrom.subtract(1, 'day'), window.timezone)} AND `
+ : ''
+ const sessionWindow = `${lowerBound}$start_timestamp <= ${tsLiteral(window.currentTo, window.timezone)}`
+ return `$session_id IN (SELECT session_id FROM sessions WHERE $channel_type = 'AI' AND ${sessionWindow})`
+}
+
+const buildAiAssistantsQuery = (window: PagePerformanceWindow, page: string | null, pathExpr: string): string => {
+ const { cur } = windowPredicates(window)
+ const pageMatch = page ? `${pageKeyExpr(pathExpr)} = ${escapeHogQLString(page)}` : 'true'
+ return `
+SELECT
+ ${AI_ASSISTANT_EXPR} AS "Assistant",
+ uniq(person_id) AS "Visitors",
+ count() AS "Views"
+FROM events
+WHERE and(event IN ${HUMAN_EVENTS}, ${aiSessionsPredicate(window)}, ${pageMatch}, (${cur}), {filters})
+GROUP BY "Assistant"
+ORDER BY "Visitors" DESC
+LIMIT 10
+`
+}
+
+export const loadAiAssistantRows = async (
+ {
+ window,
+ page,
+ pathExpr,
+ filterTestAccounts,
+ properties,
+ }: {
+ window: PagePerformanceWindow
+ page: string | null
+ pathExpr: string
+ filterTestAccounts: boolean
+ properties: WebAnalyticsPropertyFilters
+ },
+ request?: { signal: AbortSignal; queryId: string; refresh?: RefreshType }
+): Promise<AiAssistantRow[]> => {
+ const query: HogQLQuery = {
+ kind: NodeKind.HogQLQuery,
+ query: buildAiAssistantsQuery(window, page, pathExpr),
+ filters: { filterTestAccounts, properties },
+ tags: WEB_ANALYTICS_DEFAULT_QUERY_TAGS,
+ }
+ const response = await performQuery(query, { signal: request?.signal }, request?.refresh, request?.queryId)
+ return (response.results ?? []).map((row: unknown[]) => ({
+ assistant: String(row[0] ?? ''),
+ visitors: Number(row[1] ?? 0),
+ views: Number(row[2] ?? 0),
+ }))
+}
+
+export interface GoogleSearchConsoleSummary {
+ clicks: number
+ impressions: number
+ position: number | null
}
+
+export const buildGoogleSearchConsoleSummaryQuery = (
+ tableName: string,
+ page: string,
+ window: PagePerformanceWindow,
+ isPathCleaningEnabled: boolean,
+ pathCleaningFilters: { regex?: string; alias?: string }[] | undefined
+): string =>
+ `SELECT sum(clicks) AS clicks, sum(impressions) AS impressions, if(sum(impressions) > 0, sum(position * impressions) / sum(impressions), NULL) AS position
+FROM ${escapeDottedHogQLIdentifier(tableName)}
+WHERE ${gscPageAndDatePredicate(page, window, isPathCleaningEnabled, pathCleaningFilters)}`
+
+export const GOOGLE_SEARCH_CONSOLE_TABLE_NAME = 'search_analytics_by_query_page'
+
+export const findGoogleSearchConsoleTable = (tables: { name: string }[]): string | null =>
+ tables.find((table) => table.name.includes(GOOGLE_SEARCH_CONSOLE_TABLE_NAME))?.name ?? null
+
+export const GOOGLE_SEARCH_CONSOLE_DOCS_URL = 'https://posthog.com/docs/cdp/sources/google-search-console'