OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(dashboard): pages table paginates and sorts on the server
Carl-Gerhard Lindesvärdfa7d9915eb24
diff --git a/apps/start/src/components/page/pages-insights.tsx b/apps/start/src/components/page/pages-insights.tsxindex 6db4f1c76..c7362cbbe 100644--- a/apps/start/src/components/page/pages-insights.tsx+++ b/apps/start/src/components/page/pages-insights.tsx@@ -89,14 +89,14 @@ export function PagesInsights({ projectId }: PagesInsightsProps) {const analyticsQuery = useQuery(trpc.event.pages.queryOptions(- { projectId, cursor: 1, take: 1000, search: undefined, range, interval },+ { projectId, take: 1000, range, interval },{ placeholderData: keepPreviousData }));const insights = useMemo<PageInsight[]>(() => {const gscPages = gscPagesQuery.data ?? [];- const analyticsPages = analyticsQuery.data ?? [];+ const analyticsPages = analyticsQuery.data?.data ?? [];const analyticsMap = new Map(analyticsPages.map((p) => [p.origin + p.path, p])diff --git a/apps/start/src/components/pages/table/columns.tsx b/apps/start/src/components/pages/table/columns.tsxindex 17264f196..2410fef9a 100644--- a/apps/start/src/components/pages/table/columns.tsx+++ b/apps/start/src/components/pages/table/columns.tsx@@ -12,7 +12,7 @@ import type { RouterOutputs } from '@/trpc/client';// `fancyMinutes` takes seconds. Without this every page rendered "0s".const SECONDS_PER_MINUTE = 60;-export type PageRow = RouterOutputs['event']['pages'][number] & {+export type PageRow = RouterOutputs['event']['pages']['data'][number] & {gsc?: { clicks: number; impressions: number; ctr: number; position: number };};@@ -34,6 +34,7 @@ export function useColumns({id: 'page',accessorFn: (row) => `${row.origin}${row.path} ${row.title ?? ''}`,header: createHeaderColumn('Page'),+ enableSorting: false,size: 400,meta: { bold: true },cell: ({ row }) => {@@ -177,6 +178,7 @@ export function useColumns({id: 'gsc_impressions',accessorFn: (row) => row.gsc?.impressions ?? 0,header: createHeaderColumn('Impr.'),+ enableSorting: false,size: 80,cell: ({ row }) =>row.original.gsc ? (@@ -191,6 +193,7 @@ export function useColumns({id: 'gsc_ctr',accessorFn: (row) => row.gsc?.ctr ?? 0,header: createHeaderColumn('CTR'),+ enableSorting: false,size: 70,cell: ({ row }) =>row.original.gsc ? (@@ -205,6 +208,7 @@ export function useColumns({id: 'gsc_clicks',accessorFn: (row) => row.gsc?.clicks ?? 0,header: createHeaderColumn('Clicks'),+ enableSorting: false,size: 80,cell: ({ row }) =>row.original.gsc ? (diff --git a/apps/start/src/components/pages/table/index.tsx b/apps/start/src/components/pages/table/index.tsxindex 2b3549a41..a3dda5467 100644--- a/apps/start/src/components/pages/table/index.tsx+++ b/apps/start/src/components/pages/table/index.tsx@@ -1,20 +1,43 @@+import {+ TOP_PAGES_SORT_COLUMNS,+ type TopPagesSortColumn,+} from '@openpanel/core/modules/event/event.constants';import { keepPreviousData, useQuery } from '@tanstack/react-query';+import type {+ PaginationState,+ SortingState,+ Updater,+} from '@tanstack/react-table';+import { getCoreRowModel, useReactTable } from '@tanstack/react-table';+import { parseAsStringEnum, useQueryStates } from 'nuqs';import { useMemo } from 'react';import { type PageRow, useColumns } from './columns';import { OverviewInterval } from '@/components/overview/overview-interval';import { OverviewRange } from '@/components/overview/overview-range';import { useOverviewOptions } from '@/components/overview/useOverviewOptions';import { DataTable } from '@/components/ui/data-table/data-table';+import {+ useDataTableColumnVisibility,+ useDataTablePagination,+} from '@/components/ui/data-table/data-table-hooks';import {AnimatedSearchInput,DataTableToolbarContainer,} from '@/components/ui/data-table/data-table-toolbar';import { DataTableViewOptions } from '@/components/ui/data-table/data-table-view-options';-import { useTable } from '@/components/ui/data-table/use-table';import { useSearchQueryState } from '@/hooks/use-search-query-state';import { useTRPC } from '@/integrations/trpc/react';import { pushModal } from '@/modals';+const PAGE_SIZE = 50;+const SORT_ORDERS = ['asc', 'desc'] as const;+const DEFAULT_SORT = { sortBy: 'sessions', sortOrder: 'desc' } as const;+const LOADING_DATA = Array.from({ length: 9 }, () => ({})) as PageRow[];++function isSortColumn(id: string): id is TopPagesSortColumn {+ return (TOP_PAGES_SORT_COLUMNS as readonly string[]).includes(id);+}+interface PagesTableProps {projectId: string;}@@ -23,18 +46,46 @@ export function PagesTable({ projectId }: PagesTableProps) {const trpc = useTRPC();const { range, interval, startDate, endDate } = useOverviewOptions();const { debouncedSearch, setSearch, search } = useSearchQueryState();+ const {+ page,+ setPage,+ state: pagination,+ } = useDataTablePagination(PAGE_SIZE);+ const [sort, setSort] = useQueryStates(+ {+ sortBy: parseAsStringEnum([...TOP_PAGES_SORT_COLUMNS]).withDefault(+ DEFAULT_SORT.sortBy+ ),+ sortOrder: parseAsStringEnum([...SORT_ORDERS]).withDefault(+ DEFAULT_SORT.sortOrder+ ),+ },+ { clearOnDefault: true, history: 'push' }+ );+ const sorting: SortingState = [+ { id: sort.sortBy, desc: sort.sortOrder === 'desc' },+ ];const pagesQuery = useQuery(trpc.event.pages.queryOptions({projectId,- search: debouncedSearch ?? undefined,+ search: debouncedSearch || undefined,range,interval,+ cursor: page - 1,+ take: PAGE_SIZE,+ sortBy: sort.sortBy,+ sortOrder: sort.sortOrder,},{ placeholderData: keepPreviousData }));+ const pages = pagesQuery.data?.data;+ const pageKeys = useMemo(+ () => (pages ?? []).map(({ origin, path }) => ({ origin, path })),+ [pages]+ );const connectionQuery = useQuery(trpc.gsc.getConnection.queryOptions({ projectId })@@ -57,8 +108,8 @@ export function PagesTable({ projectId }: PagesTableProps) {const previousPagesQuery = useQuery(trpc.event.previousPages.queryOptions(- { projectId, range, interval },- { placeholderData: keepPreviousData }+ { projectId, range, interval, pages: pageKeys },+ { placeholderData: keepPreviousData, enabled: pageKeys.length > 0 }));@@ -87,20 +138,47 @@ export function PagesTable({ projectId }: PagesTableProps) {}, [gscPagesQuery.data]);const rawData: PageRow[] = useMemo(() => {- return (pagesQuery.data ?? []).map((p) => ({+ return (pages ?? []).map((p) => ({...p,gsc: gscMap.get(p.origin + p.path),}));- }, [pagesQuery.data, gscMap]);+ }, [pages, gscMap]);const columns = useColumns({ projectId, isGscConnected, previousMap });+ const { columnVisibility, setColumnVisibility, columnOrder, setColumnOrder } =+ useDataTableColumnVisibility(columns, 'pages');- const { table } = useTable({+ const table = useReactTable({+ data: pagesQuery.isLoading ? LOADING_DATA : rawData,columns,- data: rawData,- loading: pagesQuery.isLoading,- pageSize: 50,- name: 'pages',+ getCoreRowModel: getCoreRowModel(),+ manualPagination: true,+ manualSorting: true,+ manualFiltering: true,+ rowCount: pagesQuery.data?.meta.count,+ pageCount: Math.ceil((pagesQuery.data?.meta.count ?? 0) / PAGE_SIZE),+ filterFns: {+ isWithinRange: () => true,+ },+ state: { pagination, sorting, columnVisibility, columnOrder },+ onColumnVisibilityChange: setColumnVisibility,+ onColumnOrderChange: setColumnOrder,+ onPaginationChange: (updater: Updater<PaginationState>) => {+ const next =+ typeof updater === 'function' ? updater(pagination) : updater;+ setPage(next.pageIndex + 1);+ },+ onSortingChange: (updater: Updater<SortingState>) => {+ const [next] = typeof updater === 'function' ? updater(sorting) : updater;+ setSort(+ next && isSortColumn(next.id)+ ? { sortBy: next.id, sortOrder: next.desc ? 'desc' : 'asc' }+ : null+ );+ setPage(1);+ },+ getRowId: (row, index) =>+ row.origin === undefined ? `loading-${index}` : row.origin + row.path,});return (diff --git a/packages/core/src/modules/event/event.constants.ts b/packages/core/src/modules/event/event.constants.tsindex 9f444a05e..f3f56c8e6 100644--- a/packages/core/src/modules/event/event.constants.ts+++ b/packages/core/src/modules/event/event.constants.ts@@ -85,3 +85,12 @@ export const QUERY_EVENTS_EQUALITY_COLUMNS = [export type QueryEventsEqualityColumn =(typeof QUERY_EVENTS_EQUALITY_COLUMNS)[number];++/** The `event.pages` columns a client may sort by; the query sorts server-side. */+export const TOP_PAGES_SORT_COLUMNS = [+ 'sessions',+ 'pageviews',+ 'bounce_rate',+ 'avg_duration',+] as const;+export type TopPagesSortColumn = (typeof TOP_PAGES_SORT_COLUMNS)[number];diff --git a/packages/core/src/modules/event/event.rpc.test.ts b/packages/core/src/modules/event/event.rpc.test.tsindex 3785ed35d..4f065437d 100644--- a/packages/core/src/modules/event/event.rpc.test.ts+++ b/packages/core/src/modules/event/event.rpc.test.ts@@ -57,9 +57,9 @@ test('every query rejects an unauthenticated caller before touching a database',await expect(caller.pagesTimeseries(WINDOW)).rejects.toMatchObject(UNAUTHORIZED);- await expect(caller.previousPages(WINDOW)).rejects.toMatchObject(- UNAUTHORIZED- );+ await expect(+ caller.previousPages({ ...WINDOW, pages: [] })+ ).rejects.toMatchObject(UNAUTHORIZED);await expect(caller.pageTimeseries({ ...WINDOW, origin: 'https://a.io', path: '/' })).rejects.toMatchObject(UNAUTHORIZED);diff --git a/packages/core/src/modules/event/event.rpc.ts b/packages/core/src/modules/event/event.rpc.tsindex 63c48f6ff..32b1027e5 100644--- a/packages/core/src/modules/event/event.rpc.ts+++ b/packages/core/src/modules/event/event.rpc.ts@@ -13,6 +13,7 @@ import {zTimeInterval,} from '../report/report.constants';import { getChartStartEndDate } from '../report/src/chart-dates';+import { TOP_PAGES_SORT_COLUMNS } from './event.constants';import {getBotEventsPage,getConversionEventNames,@@ -25,6 +26,8 @@ import {} from './event.service';const DEFAULT_BOTS_LIMIT = 8;+const DEFAULT_PAGES_TAKE = 50;+const MAX_PAGES_TAKE = 1000;/*** `pagesTimeseries` returns every page in the project, so its size is the@@ -181,9 +184,16 @@ export const eventRouter = createTRPCRouter({pages: protectedProcedure.input(zChartWindow.extend({- cursor: z.number().optional(),- take: z.number().min(1).optional(),+ cursor: z.number().int().min(0).optional(),+ take: z+ .number()+ .int()+ .min(1)+ .max(MAX_PAGES_TAKE)+ .default(DEFAULT_PAGES_TAKE),search: z.string().optional(),+ sortBy: z.enum(TOP_PAGES_SORT_COLUMNS).optional(),+ sortOrder: z.enum(['asc', 'desc']).optional(),})).query(async ({ input, ctx }) => {@@ -195,13 +205,16 @@ export const eventRouter = createTRPCRouter({const { timezone } = await getSettingsForProject(ctx, input.projectId);const { startDate, endDate } = getChartStartEndDate(input, timezone);- return ctx.services.pages.getTopPages({+ return ctx.services.pages.getTopPagesPage({projectId: input.projectId,startDate,endDate,timezone,search: input.search,+ sortBy: input.sortBy,+ sortOrder: input.sortOrder,limit: input.take,+ offset: (input.cursor ?? 0) * input.take,});}),@@ -227,7 +240,13 @@ export const eventRouter = createTRPCRouter({}),previousPages: protectedProcedure- .input(zChartWindow)+ .input(+ zChartWindow.extend({+ pages: z+ .array(z.object({ origin: z.string(), path: z.string() }))+ .max(MAX_PAGES_TAKE),+ })+ ).query(async ({ input, ctx }) => {await ctx.services.auth.requireProjectAccess({userId: ctx.session.userId,@@ -235,6 +254,10 @@ export const eventRouter = createTRPCRouter({level: 'read',});+ if (input.pages.length === 0) {+ return [];+ }+const { timezone } = await getSettingsForProject(ctx, input.projectId);const { startDate, endDate } = getChartStartEndDate(input, timezone);@@ -249,6 +272,7 @@ export const eventRouter = createTRPCRouter({startDate: formatClickhouseDateTime(previousStart),endDate: formatClickhouseDateTime(previousEnd),timezone,+ pages: input.pages,});}),diff --git a/packages/core/src/modules/overview/pages.service.ts b/packages/core/src/modules/overview/pages.service.tsindex 0d6c7aecd..45a042466 100644--- a/packages/core/src/modules/overview/pages.service.ts+++ b/packages/core/src/modules/overview/pages.service.ts@@ -12,17 +12,18 @@ import {import {pageConversionsQuery,pageTimeseriesQuery,+ type TopPagesQueryInput,topPagesQuery,} from './src/pages.sql';import { runQuery } from './src/run-query';-export interface IGetPagesInput {- projectId: string;- startDate: string;- endDate: string;+export interface IGetPagesInput extends Omit<TopPagesQueryInput, 'offset'> {timezone: string;- search?: string;- limit?: number;+}++export interface IGetPagesPageInput extends IGetPagesInput {+ limit: number;+ offset: number;}export interface IPageTimeseriesRow {@@ -43,15 +44,32 @@ export interface ITopPage {bounce_rate: number;}-export function getTopPages(+type TopPageRow = ITopPage & { total_count: number };++function queryTopPages(deps: ServiceDeps,- { projectId, startDate, endDate, timezone, search, limit }: IGetPagesInput+ { timezone, ...query }: IGetPagesInput | IGetPagesPageInput+): Promise<TopPageRow[]> {+ return runQuery<TopPageRow>(deps, topPagesQuery(query), timezone);+}++export async function getTopPages(+ deps: ServiceDeps,+ input: IGetPagesInput): Promise<ITopPage[]> {- return runQuery<ITopPage>(- deps,- topPagesQuery({ projectId, startDate, endDate, search, limit }),- timezone- );+ const rows = await queryTopPages(deps, input);+ return rows.map(({ total_count: _total, ...page }) => page);+}++export async function getTopPagesPage(+ deps: ServiceDeps,+ input: IGetPagesPageInput+): Promise<{ data: ITopPage[]; meta: { count: number; take: number } }> {+ const rows = await queryTopPages(deps, input);+ return {+ data: rows.map(({ total_count: _total, ...page }) => page),+ meta: { count: rows[0]?.total_count ?? 0, take: input.limit },+ };}export interface IGetPageTimeseriesInput extends IGetPagesInput {@@ -222,6 +240,9 @@ export function createPagesService(return {getTopPages: (input: IGetPagesInput): Promise<ITopPage[]> =>getTopPages(deps, input),+ getTopPagesPage: (+ input: IGetPagesPageInput+ ): ReturnType<typeof getTopPagesPage> => getTopPagesPage(deps, input),getPageTimeseries: (input: IGetPageTimeseriesInput): Promise<IPageTimeseriesRow[]> => getPageTimeseries(deps, input),diff --git a/packages/core/src/modules/overview/src/pages.sql.test.ts b/packages/core/src/modules/overview/src/pages.sql.test.tsindex 86cc4b75f..ddc2bd5ee 100644--- a/packages/core/src/modules/overview/src/pages.sql.test.ts+++ b/packages/core/src/modules/overview/src/pages.sql.test.ts@@ -57,6 +57,38 @@ describe('pages.sql — parses against ClickHouse', () => {);});+ it('topPagesQuery — sorted, offset, restricted to given pages', async () => {+ const fragment = PG.topPagesQuery({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ sortBy: 'bounce_rate',+ sortOrder: 'asc',+ limit: 50,+ offset: 100,+ pages: [{ origin: 'https://a.test', path: '/pricing' }],+ });+ await explain(fragment);+ const { query, query_params } = fragment.toStatement();+ expect(query).toContain('ORDER BY bounce_rate ASC, origin, path');+ expect(query).toMatch(/LIMIT \{p\d+:UInt64\}\s+OFFSET \{p\d+:UInt64\}/);+ expect(query).toContain('count() OVER () as total_count');+ expect(Object.values(query_params)).toContainEqual([+ 'https://a.test/pricing',+ ]);+ });++ it('topPagesQuery — rejects a sort column outside the whitelist', () => {+ expect(() =>+ PG.topPagesQuery({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ sortBy: 'title' as never,+ })+ ).toThrow();+ });+it('topPagesQuery — page_titles is bounded to the requested range', () => {const { query, query_params } = PG.topPagesQuery({projectId: PROJECT_ID,diff --git a/packages/core/src/modules/overview/src/pages.sql.ts b/packages/core/src/modules/overview/src/pages.sql.tsindex e54d9c709..f28a08374 100644--- a/packages/core/src/modules/overview/src/pages.sql.ts+++ b/packages/core/src/modules/overview/src/pages.sql.ts@@ -3,6 +3,10 @@import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import { toRangeBoundaryLiteral } from '../../../shared/ch-dates';+import {+ TOP_PAGES_SORT_COLUMNS,+ type TopPagesSortColumn,+} from '../../event/event.constants';import type { IInterval } from '../../report/report.constants';import { fillClause, toStartOf } from './overview.sql';@@ -19,7 +23,20 @@ export interface TopPagesQueryInput {startDate: string;endDate: string;search?: string;+ /** Only these pages; the trend column asks for the previous period of one page of rows. */+ pages?: readonly { origin: string; path: string }[];+ sortBy?: TopPagesSortColumn;+ sortOrder?: 'asc' | 'desc';limit?: number;+ offset?: number;+}++function pagesClause(pages: TopPagesQueryInput['pages']): SqlFragment {+ if (!pages) {+ return sql.empty;+ }+ const keys = pages.map((page) => page.origin + page.path);+ return sql`AND concat(e.origin, e.path) IN ${sql.array('String', keys)}`;}function searchClause(search: string | undefined): SqlFragment {@@ -70,7 +87,14 @@ export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {input.limit === undefined? sql.empty: sql`LIMIT ${sql.uint64(input.limit)}`;+ const offsetClause = input.offset+ ? sql`OFFSET ${sql.uint64(input.offset)}`+ : sql.empty;+ const sortColumn = sql.id(input.sortBy ?? 'sessions', TOP_PAGES_SORT_COLUMNS);+ const sortDirection = input.sortOrder === 'asc' ? sql`ASC` : sql`DESC`;+ // `count() OVER ()` runs after GROUP BY and before LIMIT: the number of pages+ // in the window, so one query both pages and counts.return sql`WITHpage_titles AS (${titlesCte}),@@ -86,15 +110,18 @@ export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {(uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /nullIf(uniq(e.session_id), 0),2- ) as bounce_rate+ ) as bounce_rate,+ count() OVER () as total_countFROM screen_view_durations eLEFT JOIN (${sessionsSubquery}) s ON e.session_id = s.id AND e.project_id = s.project_idLEFT JOIN page_titles pt ON concat(e.origin, e.path) = pt.page_keyWHERE 1${searchClause(input.search)}+ ${pagesClause(input.pages)}GROUP BY e.origin, e.path, pt.title- ORDER BY sessions DESC+ ORDER BY ${sortColumn} ${sortDirection}, origin, path${limitClause}+ ${offsetClause}`;}