Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(dashboard): pages table paginates and sorts on the server

Carl-Gerhard Lindesvärdfa7d9915eb24
diff --git a/apps/start/src/components/page/pages-insights.tsx b/apps/start/src/components/page/pages-insights.tsx
index 6db4f1c76..c7362cbbe 100644
--- a/apps/start/src/components/page/pages-insights.tsx
+++ b/apps/start/src/components/page/pages-insights.tsx
@@ -89,14 +89,14 @@ export function PagesInsights({ projectId }: PagesInsightsProps) {
const analyticsQuery = useQuery(
trpc.event.pages.queryOptions(
- { projectId, cursor: 1, take: 1000, search: undefined, range, interval },
+ { projectId, take: 1000, range, interval },
{ placeholderData: keepPreviousData }
)
);
const insights = useMemo<PageInsight[]>(() => {
const gscPages = gscPagesQuery.data ?? [];
- const analyticsPages = analyticsQuery.data ?? [];
+ const analyticsPages = analyticsQuery.data?.data ?? [];
const analyticsMap = new Map(
analyticsPages.map((p) => [p.origin + p.path, p])
diff --git a/apps/start/src/components/pages/table/columns.tsx b/apps/start/src/components/pages/table/columns.tsx
index 17264f196..2410fef9a 100644
--- a/apps/start/src/components/pages/table/columns.tsx
+++ b/apps/start/src/components/pages/table/columns.tsx
@@ -12,7 +12,7 @@ import type { RouterOutputs } from '@/trpc/client';
// `fancyMinutes` takes seconds. Without this every page rendered "0s".
const SECONDS_PER_MINUTE = 60;
-export type PageRow = RouterOutputs['event']['pages'][number] & {
+export type PageRow = RouterOutputs['event']['pages']['data'][number] & {
gsc?: { clicks: number; impressions: number; ctr: number; position: number };
};
@@ -34,6 +34,7 @@ export function useColumns({
id: 'page',
accessorFn: (row) => `${row.origin}${row.path} ${row.title ?? ''}`,
header: createHeaderColumn('Page'),
+ enableSorting: false,
size: 400,
meta: { bold: true },
cell: ({ row }) => {
@@ -177,6 +178,7 @@ export function useColumns({
id: 'gsc_impressions',
accessorFn: (row) => row.gsc?.impressions ?? 0,
header: createHeaderColumn('Impr.'),
+ enableSorting: false,
size: 80,
cell: ({ row }) =>
row.original.gsc ? (
@@ -191,6 +193,7 @@ export function useColumns({
id: 'gsc_ctr',
accessorFn: (row) => row.gsc?.ctr ?? 0,
header: createHeaderColumn('CTR'),
+ enableSorting: false,
size: 70,
cell: ({ row }) =>
row.original.gsc ? (
@@ -205,6 +208,7 @@ export function useColumns({
id: 'gsc_clicks',
accessorFn: (row) => row.gsc?.clicks ?? 0,
header: createHeaderColumn('Clicks'),
+ enableSorting: false,
size: 80,
cell: ({ row }) =>
row.original.gsc ? (
diff --git a/apps/start/src/components/pages/table/index.tsx b/apps/start/src/components/pages/table/index.tsx
index 2b3549a41..a3dda5467 100644
--- a/apps/start/src/components/pages/table/index.tsx
+++ b/apps/start/src/components/pages/table/index.tsx
@@ -1,20 +1,43 @@
+import {
+ TOP_PAGES_SORT_COLUMNS,
+ type TopPagesSortColumn,
+} from '@openpanel/core/modules/event/event.constants';
import { keepPreviousData, useQuery } from '@tanstack/react-query';
+import type {
+ PaginationState,
+ SortingState,
+ Updater,
+} from '@tanstack/react-table';
+import { getCoreRowModel, useReactTable } from '@tanstack/react-table';
+import { parseAsStringEnum, useQueryStates } from 'nuqs';
import { useMemo } from 'react';
import { type PageRow, useColumns } from './columns';
import { OverviewInterval } from '@/components/overview/overview-interval';
import { OverviewRange } from '@/components/overview/overview-range';
import { useOverviewOptions } from '@/components/overview/useOverviewOptions';
import { DataTable } from '@/components/ui/data-table/data-table';
+import {
+ useDataTableColumnVisibility,
+ useDataTablePagination,
+} from '@/components/ui/data-table/data-table-hooks';
import {
AnimatedSearchInput,
DataTableToolbarContainer,
} from '@/components/ui/data-table/data-table-toolbar';
import { DataTableViewOptions } from '@/components/ui/data-table/data-table-view-options';
-import { useTable } from '@/components/ui/data-table/use-table';
import { useSearchQueryState } from '@/hooks/use-search-query-state';
import { useTRPC } from '@/integrations/trpc/react';
import { pushModal } from '@/modals';
+const PAGE_SIZE = 50;
+const SORT_ORDERS = ['asc', 'desc'] as const;
+const DEFAULT_SORT = { sortBy: 'sessions', sortOrder: 'desc' } as const;
+const LOADING_DATA = Array.from({ length: 9 }, () => ({})) as PageRow[];
+
+function isSortColumn(id: string): id is TopPagesSortColumn {
+ return (TOP_PAGES_SORT_COLUMNS as readonly string[]).includes(id);
+}
+
interface PagesTableProps {
projectId: string;
}
@@ -23,18 +46,46 @@ export function PagesTable({ projectId }: PagesTableProps) {
const trpc = useTRPC();
const { range, interval, startDate, endDate } = useOverviewOptions();
const { debouncedSearch, setSearch, search } = useSearchQueryState();
+ const {
+ page,
+ setPage,
+ state: pagination,
+ } = useDataTablePagination(PAGE_SIZE);
+ const [sort, setSort] = useQueryStates(
+ {
+ sortBy: parseAsStringEnum([...TOP_PAGES_SORT_COLUMNS]).withDefault(
+ DEFAULT_SORT.sortBy
+ ),
+ sortOrder: parseAsStringEnum([...SORT_ORDERS]).withDefault(
+ DEFAULT_SORT.sortOrder
+ ),
+ },
+ { clearOnDefault: true, history: 'push' }
+ );
+ const sorting: SortingState = [
+ { id: sort.sortBy, desc: sort.sortOrder === 'desc' },
+ ];
const pagesQuery = useQuery(
trpc.event.pages.queryOptions(
{
projectId,
- search: debouncedSearch ?? undefined,
+ search: debouncedSearch || undefined,
range,
interval,
+ cursor: page - 1,
+ take: PAGE_SIZE,
+ sortBy: sort.sortBy,
+ sortOrder: sort.sortOrder,
},
{ placeholderData: keepPreviousData }
)
);
+ const pages = pagesQuery.data?.data;
+ const pageKeys = useMemo(
+ () => (pages ?? []).map(({ origin, path }) => ({ origin, path })),
+ [pages]
+ );
const connectionQuery = useQuery(
trpc.gsc.getConnection.queryOptions({ projectId })
@@ -57,8 +108,8 @@ export function PagesTable({ projectId }: PagesTableProps) {
const previousPagesQuery = useQuery(
trpc.event.previousPages.queryOptions(
- { projectId, range, interval },
- { placeholderData: keepPreviousData }
+ { projectId, range, interval, pages: pageKeys },
+ { placeholderData: keepPreviousData, enabled: pageKeys.length > 0 }
)
);
@@ -87,20 +138,47 @@ export function PagesTable({ projectId }: PagesTableProps) {
}, [gscPagesQuery.data]);
const rawData: PageRow[] = useMemo(() => {
- return (pagesQuery.data ?? []).map((p) => ({
+ return (pages ?? []).map((p) => ({
...p,
gsc: gscMap.get(p.origin + p.path),
}));
- }, [pagesQuery.data, gscMap]);
+ }, [pages, gscMap]);
const columns = useColumns({ projectId, isGscConnected, previousMap });
+ const { columnVisibility, setColumnVisibility, columnOrder, setColumnOrder } =
+ useDataTableColumnVisibility(columns, 'pages');
- const { table } = useTable({
+ const table = useReactTable({
+ data: pagesQuery.isLoading ? LOADING_DATA : rawData,
columns,
- data: rawData,
- loading: pagesQuery.isLoading,
- pageSize: 50,
- name: 'pages',
+ getCoreRowModel: getCoreRowModel(),
+ manualPagination: true,
+ manualSorting: true,
+ manualFiltering: true,
+ rowCount: pagesQuery.data?.meta.count,
+ pageCount: Math.ceil((pagesQuery.data?.meta.count ?? 0) / PAGE_SIZE),
+ filterFns: {
+ isWithinRange: () => true,
+ },
+ state: { pagination, sorting, columnVisibility, columnOrder },
+ onColumnVisibilityChange: setColumnVisibility,
+ onColumnOrderChange: setColumnOrder,
+ onPaginationChange: (updater: Updater<PaginationState>) => {
+ const next =
+ typeof updater === 'function' ? updater(pagination) : updater;
+ setPage(next.pageIndex + 1);
+ },
+ onSortingChange: (updater: Updater<SortingState>) => {
+ const [next] = typeof updater === 'function' ? updater(sorting) : updater;
+ setSort(
+ next && isSortColumn(next.id)
+ ? { sortBy: next.id, sortOrder: next.desc ? 'desc' : 'asc' }
+ : null
+ );
+ setPage(1);
+ },
+ getRowId: (row, index) =>
+ row.origin === undefined ? `loading-${index}` : row.origin + row.path,
});
return (
diff --git a/packages/core/src/modules/event/event.constants.ts b/packages/core/src/modules/event/event.constants.ts
index 9f444a05e..f3f56c8e6 100644
--- a/packages/core/src/modules/event/event.constants.ts
+++ b/packages/core/src/modules/event/event.constants.ts
@@ -85,3 +85,12 @@ export const QUERY_EVENTS_EQUALITY_COLUMNS = [
export type QueryEventsEqualityColumn =
(typeof QUERY_EVENTS_EQUALITY_COLUMNS)[number];
+
+/** The `event.pages` columns a client may sort by; the query sorts server-side. */
+export const TOP_PAGES_SORT_COLUMNS = [
+ 'sessions',
+ 'pageviews',
+ 'bounce_rate',
+ 'avg_duration',
+] as const;
+export type TopPagesSortColumn = (typeof TOP_PAGES_SORT_COLUMNS)[number];
diff --git a/packages/core/src/modules/event/event.rpc.test.ts b/packages/core/src/modules/event/event.rpc.test.ts
index 3785ed35d..4f065437d 100644
--- a/packages/core/src/modules/event/event.rpc.test.ts
+++ b/packages/core/src/modules/event/event.rpc.test.ts
@@ -57,9 +57,9 @@ test('every query rejects an unauthenticated caller before touching a database',
await expect(caller.pagesTimeseries(WINDOW)).rejects.toMatchObject(
UNAUTHORIZED
);
- await expect(caller.previousPages(WINDOW)).rejects.toMatchObject(
- UNAUTHORIZED
- );
+ await expect(
+ caller.previousPages({ ...WINDOW, pages: [] })
+ ).rejects.toMatchObject(UNAUTHORIZED);
await expect(
caller.pageTimeseries({ ...WINDOW, origin: 'https://a.io', path: '/' })
).rejects.toMatchObject(UNAUTHORIZED);
diff --git a/packages/core/src/modules/event/event.rpc.ts b/packages/core/src/modules/event/event.rpc.ts
index 63c48f6ff..32b1027e5 100644
--- a/packages/core/src/modules/event/event.rpc.ts
+++ b/packages/core/src/modules/event/event.rpc.ts
@@ -13,6 +13,7 @@ import {
zTimeInterval,
} from '../report/report.constants';
import { getChartStartEndDate } from '../report/src/chart-dates';
+import { TOP_PAGES_SORT_COLUMNS } from './event.constants';
import {
getBotEventsPage,
getConversionEventNames,
@@ -25,6 +26,8 @@ import {
} from './event.service';
const DEFAULT_BOTS_LIMIT = 8;
+const DEFAULT_PAGES_TAKE = 50;
+const MAX_PAGES_TAKE = 1000;
/**
* `pagesTimeseries` returns every page in the project, so its size is the
@@ -181,9 +184,16 @@ export const eventRouter = createTRPCRouter({
pages: protectedProcedure
.input(
zChartWindow.extend({
- cursor: z.number().optional(),
- take: z.number().min(1).optional(),
+ cursor: z.number().int().min(0).optional(),
+ take: z
+ .number()
+ .int()
+ .min(1)
+ .max(MAX_PAGES_TAKE)
+ .default(DEFAULT_PAGES_TAKE),
search: z.string().optional(),
+ sortBy: z.enum(TOP_PAGES_SORT_COLUMNS).optional(),
+ sortOrder: z.enum(['asc', 'desc']).optional(),
})
)
.query(async ({ input, ctx }) => {
@@ -195,13 +205,16 @@ export const eventRouter = createTRPCRouter({
const { timezone } = await getSettingsForProject(ctx, input.projectId);
const { startDate, endDate } = getChartStartEndDate(input, timezone);
- return ctx.services.pages.getTopPages({
+ return ctx.services.pages.getTopPagesPage({
projectId: input.projectId,
startDate,
endDate,
timezone,
search: input.search,
+ sortBy: input.sortBy,
+ sortOrder: input.sortOrder,
limit: input.take,
+ offset: (input.cursor ?? 0) * input.take,
});
}),
@@ -227,7 +240,13 @@ export const eventRouter = createTRPCRouter({
}),
previousPages: protectedProcedure
- .input(zChartWindow)
+ .input(
+ zChartWindow.extend({
+ pages: z
+ .array(z.object({ origin: z.string(), path: z.string() }))
+ .max(MAX_PAGES_TAKE),
+ })
+ )
.query(async ({ input, ctx }) => {
await ctx.services.auth.requireProjectAccess({
userId: ctx.session.userId,
@@ -235,6 +254,10 @@ export const eventRouter = createTRPCRouter({
level: 'read',
});
+ if (input.pages.length === 0) {
+ return [];
+ }
+
const { timezone } = await getSettingsForProject(ctx, input.projectId);
const { startDate, endDate } = getChartStartEndDate(input, timezone);
@@ -249,6 +272,7 @@ export const eventRouter = createTRPCRouter({
startDate: formatClickhouseDateTime(previousStart),
endDate: formatClickhouseDateTime(previousEnd),
timezone,
+ pages: input.pages,
});
}),
diff --git a/packages/core/src/modules/overview/pages.service.ts b/packages/core/src/modules/overview/pages.service.ts
index 0d6c7aecd..45a042466 100644
--- a/packages/core/src/modules/overview/pages.service.ts
+++ b/packages/core/src/modules/overview/pages.service.ts
@@ -12,17 +12,18 @@ import {
import {
pageConversionsQuery,
pageTimeseriesQuery,
+ type TopPagesQueryInput,
topPagesQuery,
} from './src/pages.sql';
import { runQuery } from './src/run-query';
-export interface IGetPagesInput {
- projectId: string;
- startDate: string;
- endDate: string;
+export interface IGetPagesInput extends Omit<TopPagesQueryInput, 'offset'> {
timezone: string;
- search?: string;
- limit?: number;
+}
+
+export interface IGetPagesPageInput extends IGetPagesInput {
+ limit: number;
+ offset: number;
}
export interface IPageTimeseriesRow {
@@ -43,15 +44,32 @@ export interface ITopPage {
bounce_rate: number;
}
-export function getTopPages(
+type TopPageRow = ITopPage & { total_count: number };
+
+function queryTopPages(
deps: ServiceDeps,
- { projectId, startDate, endDate, timezone, search, limit }: IGetPagesInput
+ { timezone, ...query }: IGetPagesInput | IGetPagesPageInput
+): Promise<TopPageRow[]> {
+ return runQuery<TopPageRow>(deps, topPagesQuery(query), timezone);
+}
+
+export async function getTopPages(
+ deps: ServiceDeps,
+ input: IGetPagesInput
): Promise<ITopPage[]> {
- return runQuery<ITopPage>(
- deps,
- topPagesQuery({ projectId, startDate, endDate, search, limit }),
- timezone
- );
+ const rows = await queryTopPages(deps, input);
+ return rows.map(({ total_count: _total, ...page }) => page);
+}
+
+export async function getTopPagesPage(
+ deps: ServiceDeps,
+ input: IGetPagesPageInput
+): Promise<{ data: ITopPage[]; meta: { count: number; take: number } }> {
+ const rows = await queryTopPages(deps, input);
+ return {
+ data: rows.map(({ total_count: _total, ...page }) => page),
+ meta: { count: rows[0]?.total_count ?? 0, take: input.limit },
+ };
}
export interface IGetPageTimeseriesInput extends IGetPagesInput {
@@ -222,6 +240,9 @@ export function createPagesService(
return {
getTopPages: (input: IGetPagesInput): Promise<ITopPage[]> =>
getTopPages(deps, input),
+ getTopPagesPage: (
+ input: IGetPagesPageInput
+ ): ReturnType<typeof getTopPagesPage> => getTopPagesPage(deps, input),
getPageTimeseries: (
input: IGetPageTimeseriesInput
): Promise<IPageTimeseriesRow[]> => getPageTimeseries(deps, input),
diff --git a/packages/core/src/modules/overview/src/pages.sql.test.ts b/packages/core/src/modules/overview/src/pages.sql.test.ts
index 86cc4b75f..ddc2bd5ee 100644
--- a/packages/core/src/modules/overview/src/pages.sql.test.ts
+++ b/packages/core/src/modules/overview/src/pages.sql.test.ts
@@ -57,6 +57,38 @@ describe('pages.sql — parses against ClickHouse', () => {
);
});
+ it('topPagesQuery — sorted, offset, restricted to given pages', async () => {
+ const fragment = PG.topPagesQuery({
+ projectId: PROJECT_ID,
+ startDate: START,
+ endDate: END,
+ sortBy: 'bounce_rate',
+ sortOrder: 'asc',
+ limit: 50,
+ offset: 100,
+ pages: [{ origin: 'https://a.test', path: '/pricing' }],
+ });
+ await explain(fragment);
+ const { query, query_params } = fragment.toStatement();
+ expect(query).toContain('ORDER BY bounce_rate ASC, origin, path');
+ expect(query).toMatch(/LIMIT \{p\d+:UInt64\}\s+OFFSET \{p\d+:UInt64\}/);
+ expect(query).toContain('count() OVER () as total_count');
+ expect(Object.values(query_params)).toContainEqual([
+ 'https://a.test/pricing',
+ ]);
+ });
+
+ it('topPagesQuery — rejects a sort column outside the whitelist', () => {
+ expect(() =>
+ PG.topPagesQuery({
+ projectId: PROJECT_ID,
+ startDate: START,
+ endDate: END,
+ sortBy: 'title' as never,
+ })
+ ).toThrow();
+ });
+
it('topPagesQuery — page_titles is bounded to the requested range', () => {
const { query, query_params } = PG.topPagesQuery({
projectId: PROJECT_ID,
diff --git a/packages/core/src/modules/overview/src/pages.sql.ts b/packages/core/src/modules/overview/src/pages.sql.ts
index e54d9c709..f28a08374 100644
--- a/packages/core/src/modules/overview/src/pages.sql.ts
+++ b/packages/core/src/modules/overview/src/pages.sql.ts
@@ -3,6 +3,10 @@
import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
import { toRangeBoundaryLiteral } from '../../../shared/ch-dates';
+import {
+ TOP_PAGES_SORT_COLUMNS,
+ type TopPagesSortColumn,
+} from '../../event/event.constants';
import type { IInterval } from '../../report/report.constants';
import { fillClause, toStartOf } from './overview.sql';
@@ -19,7 +23,20 @@ export interface TopPagesQueryInput {
startDate: string;
endDate: string;
search?: string;
+ /** Only these pages; the trend column asks for the previous period of one page of rows. */
+ pages?: readonly { origin: string; path: string }[];
+ sortBy?: TopPagesSortColumn;
+ sortOrder?: 'asc' | 'desc';
limit?: number;
+ offset?: number;
+}
+
+function pagesClause(pages: TopPagesQueryInput['pages']): SqlFragment {
+ if (!pages) {
+ return sql.empty;
+ }
+ const keys = pages.map((page) => page.origin + page.path);
+ return sql`AND concat(e.origin, e.path) IN ${sql.array('String', keys)}`;
}
function searchClause(search: string | undefined): SqlFragment {
@@ -70,7 +87,14 @@ export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {
input.limit === undefined
? sql.empty
: sql`LIMIT ${sql.uint64(input.limit)}`;
+ const offsetClause = input.offset
+ ? sql`OFFSET ${sql.uint64(input.offset)}`
+ : sql.empty;
+ const sortColumn = sql.id(input.sortBy ?? 'sessions', TOP_PAGES_SORT_COLUMNS);
+ const sortDirection = input.sortOrder === 'asc' ? sql`ASC` : sql`DESC`;
+ // `count() OVER ()` runs after GROUP BY and before LIMIT: the number of pages
+ // in the window, so one query both pages and counts.
return sql`
WITH
page_titles AS (${titlesCte}),
@@ -86,15 +110,18 @@ export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {
(uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /
nullIf(uniq(e.session_id), 0),
2
- ) as bounce_rate
+ ) as bounce_rate,
+ count() OVER () as total_count
FROM screen_view_durations e
LEFT JOIN (${sessionsSubquery}) s ON e.session_id = s.id AND e.project_id = s.project_id
LEFT JOIN page_titles pt ON concat(e.origin, e.path) = pt.page_key
WHERE 1
${searchClause(input.search)}
+ ${pagesClause(input.pages)}
GROUP BY e.origin, e.path, pt.title
- ORDER BY sessions DESC
+ ORDER BY ${sortColumn} ${sortDirection}, origin, path
${limitClause}
+ ${offsetClause}
`;
}