Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor analytics components and API for enhanced data handling

Bill Yangb248cee63b35
diff --git a/client/src/api/analytics/useSingleCol.ts b/client/src/api/analytics/useSingleCol.ts
index 89e6faddc..b99d40296 100644
--- a/client/src/api/analytics/useSingleCol.ts
+++ b/client/src/api/analytics/useSingleCol.ts
@@ -10,13 +10,14 @@ export type SingleColResponse = {
value: string;
count: number;
percentage: number;
- avg_session_duration?: number;
- bounce_rate?: number;
+ pageviews?: number;
+ pageviews_percentage?: number;
+ time_on_page_seconds?: number;
};
export function useSingleCol({
parameter,
- limit = 1000,
+ limit = 100,
periodTime,
useFilters = true,
}: {
diff --git a/client/src/app/[site]/components/shared/StandardSection/BaseStandardSectionDialog.tsx b/client/src/app/[site]/components/shared/StandardSection/BaseStandardSectionDialog.tsx
index b72fb87c9..5808a9741 100644
--- a/client/src/app/[site]/components/shared/StandardSection/BaseStandardSectionDialog.tsx
+++ b/client/src/app/[site]/components/shared/StandardSection/BaseStandardSectionDialog.tsx
@@ -1,3 +1,4 @@
+import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
@@ -5,13 +6,20 @@ import {
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
-import { Button } from "@/components/ui/button";
-import { round } from "lodash";
-import { SquareArrowOutUpRight, Search } from "lucide-react";
-import { ReactNode, useState } from "react";
+import { Input } from "@/components/ui/input";
+import {
+ createColumnHelper,
+ flexRender,
+ getCoreRowModel,
+ getSortedRowModel,
+ useReactTable,
+} from "@tanstack/react-table";
+import { useDebounce } from "@uidotdev/usehooks";
+import { Search, SquareArrowOutUpRight } from "lucide-react";
+import { ReactNode, useMemo, useState } from "react";
import { SingleColResponse } from "../../../../../api/analytics/useSingleCol";
import { addFilter, FilterParameter } from "../../../../../lib/store";
-import { Input } from "@/components/ui/input";
+import { formatSecondsAsMinutesAndSeconds } from "../../../../../lib/utils";
interface BaseStandardSectionDialogProps {
title: string;
@@ -26,6 +34,8 @@ interface BaseStandardSectionDialogProps {
filterParameter: FilterParameter;
}
+const columnHelper = createColumnHelper<SingleColResponse>();
+
export function BaseStandardSectionDialog({
title,
data,
@@ -39,19 +49,96 @@ export function BaseStandardSectionDialog({
filterParameter,
}: BaseStandardSectionDialogProps) {
const [searchTerm, setSearchTerm] = useState("");
+ const debouncedSearchTerm = useDebounce(searchTerm, 200);
if (!data || data.length === 0) return null;
const labelFnToUse = getFilterLabel || getValue;
// Filter data based on search term
- const filteredData = data.filter((item) => {
- const label =
- typeof labelFnToUse(item) === "string"
- ? (labelFnToUse(item) as string)
- : labelFnToUse(item);
+ const filteredData = useMemo(
+ () =>
+ data.filter((item) => {
+ const label =
+ typeof labelFnToUse(item) === "string"
+ ? (labelFnToUse(item) as string)
+ : labelFnToUse(item);
+
+ return String(label)
+ .toLowerCase()
+ .includes(debouncedSearchTerm.toLowerCase());
+ }),
+ [data, labelFnToUse, debouncedSearchTerm]
+ );
- return String(label).toLowerCase().includes(searchTerm.toLowerCase());
+ const columns = useMemo(() => {
+ const cols = [
+ columnHelper.accessor("value", {
+ header: title,
+ cell: ({ row }) => (
+ <div className="flex flex-row gap-1 items-center text-left">
+ {getLabel(row.original)}
+ {getLink && (
+ <a
+ href={getLink(row.original)}
+ target="_blank"
+ onClick={(e) => e.stopPropagation()}
+ >
+ <SquareArrowOutUpRight
+ className="w-3 h-3 text-neutral-300 hover:text-neutral-100"
+ strokeWidth={3}
+ />
+ </a>
+ )}
+ </div>
+ ),
+ }),
+ columnHelper.accessor("count", {
+ header: "Sessions",
+ cell: (info) => (
+ <div className="text-left">{info.getValue().toLocaleString()}</div>
+ ),
+ }),
+ ];
+
+ const hasPageviews = data?.[0]?.pageviews !== undefined;
+ if (hasPageviews) {
+ cols.push(
+ columnHelper.accessor("pageviews", {
+ header: "Pageviews",
+ cell: (info) => (
+ <div className="text-left">{info.getValue()?.toLocaleString()}</div>
+ ),
+ }) as any
+ );
+ }
+
+ const hasDuration = data?.[0]?.time_on_page_seconds !== undefined;
+ if (hasDuration) {
+ cols.push(
+ columnHelper.accessor("time_on_page_seconds", {
+ header: "Duration",
+ cell: (info) => (
+ <div className="text-left">
+ {formatSecondsAsMinutesAndSeconds(
+ Math.round(info.getValue() ?? 0)
+ )}
+ </div>
+ ),
+ }) as any
+ );
+ }
+ return cols;
+ }, []);
+
+ // Set up table instance
+ const table = useReactTable({
+ data: filteredData || [],
+ columns,
+ getCoreRowModel: getCoreRowModel(),
+ getSortedRowModel: getSortedRowModel(),
+ manualSorting: true,
+ sortDescFirst: true,
});
return (
@@ -63,71 +150,71 @@ export function BaseStandardSectionDialog({
<DialogHeader>
<DialogTitle>{title}</DialogTitle>
</DialogHeader>
- <div className="flex flex-col gap-2 max-h-[calc(80vh-80px)] overflow-hidden ">
+ <div className="flex flex-col gap-2 ">
<div className="relative">
<Search className="absolute left-2.5 top-2.5 h-4 w-4 text-neutral-400" />
<Input
type="text"
placeholder={`Filter ${data.length} items...`}
- className="pl-9 bg-neutral-900 border-neutral-700 focus-visible:ring-accent-400"
+ className="pl-9 bg-neutral-900 border-neutral-700 focus-visible:ring-accent-400 text-xs"
value={searchTerm}
onChange={(e) => setSearchTerm(e.target.value)}
/>
</div>
- <div className="flex flex-row gap-2 justify-between text-sm text-neutral-400 pr-4 md:pr-20">
- <div>{title}</div>
- <div>{countLabel || "Sessions"}</div>
- </div>
- <div className="flex flex-col gap-2 overflow-y-auto overflow-x-hidden max-h-[calc(80vh-150px)]">
- {filteredData.length > 0 ? (
- filteredData.map((e) => (
- <div
- key={getKey(e)}
- className="relative flex items-center mr-1 cursor-pointer hover:bg-neutral-850"
- onClick={() =>
- addFilter({
- parameter: filterParameter,
- value: [getValue(e)],
- type: "equals",
- })
- }
- >
- <div
- className="absolute inset-0 bg-accent-400 py-2 opacity-30 rounded-md"
- style={{ width: `${Math.min(e.percentage * ratio, 100)}%` }}
- ></div>
- <div className="z-10 ml-2 mr-2 md:mr-4 flex justify-between items-center text-xs w-full min-h-6 py-1">
- <div className="flex items-center gap-1 min-w-0 max-w-[65%]">
- <div className="truncate">{getLabel(e)}</div>
- {getLink && (
- <a
- href={getLink(e)}
- target="_blank"
- onClick={(e) => e.stopPropagation()}
- className="flex-shrink-0"
- >
- <SquareArrowOutUpRight
- className="w-3 h-3 text-neutral-300 hover:text-neutral-100"
- strokeWidth={3}
- />
- </a>
- )}
- </div>
- <div className="flex flex-shrink-0">
- <div>{e.count.toLocaleString()}</div>
- <div className="mx-2 bg-neutral-400 w-[1px] rounded-full h-5"></div>
- <div className="w-10 text-neutral-400">
- {round(e.percentage, 2)}%
- </div>
- </div>
- </div>
- </div>
- ))
- ) : (
- <div className="py-10 text-center text-neutral-400">
- No results match your search
- </div>
- )}
+ <div className="max-h-[80vh] overflow-y-auto">
+ <table className="w-full text-xs text-left">
+ <thead className="bg-neutral-900 text-neutral-400 sticky top-0 z-10">
+ {table.getHeaderGroups().map((headerGroup) => (
+ <tr key={headerGroup.id}>
+ {headerGroup.headers.map((header) => (
+ <th
+ key={header.id}
+ scope="col"
+ className="px-3 py-1 font-medium whitespace-nowrap"
+ style={{
+ minWidth: header.id === "user_id" ? "100px" : "auto",
+ }}
+ >
+ {header.isPlaceholder
+ ? null
+ : flexRender(
+ header.column.columnDef.header,
+ header.getContext()
+ )}
+ </th>
+ ))}
+ </tr>
+ ))}
+ </thead>
+ <tbody>
+ {table.getRowModel().rows.map((row) => {
+ return (
+ <tr
+ key={row.id}
+ className="border-b border-neutral-800 hover:bg-neutral-850 cursor-pointer group"
+ onClick={() =>
+ addFilter({
+ parameter: filterParameter,
+ value: [getValue(row.original)],
+ type: "equals",
+ })
+ }
+ >
+ {row.getVisibleCells().map((cell) => (
+ <td key={cell.id} className="px-3 py-2 relative">
+ <span className="relative z-0">
+ {flexRender(
+ cell.column.columnDef.cell,
+ cell.getContext()
+ )}
+ </span>
+ </td>
+ ))}
+ </tr>
+ );
+ })}
+ </tbody>
+ </table>
</div>
</div>
</DialogContent>
diff --git a/client/src/app/[site]/components/shared/StandardSection/StandardSectionRealtime.tsx b/client/src/app/[site]/components/shared/StandardSection/StandardSectionRealtime.tsx
index bc1e4444a..6c0c6b507 100644
--- a/client/src/app/[site]/components/shared/StandardSection/StandardSectionRealtime.tsx
+++ b/client/src/app/[site]/components/shared/StandardSection/StandardSectionRealtime.tsx
@@ -30,19 +30,18 @@ export function StandardSectionRealtime({
parameter: filterParameter,
});
- return (
- <BaseStandardSection
- title={title}
- data={data}
- isFetching={isFetching}
- error={error}
- refetch={refetch}
- getKey={getKey}
- getLabel={getLabel}
- getValue={getValue}
- getLink={getLink}
- countLabel={countLabel}
- filterParameter={filterParameter}
- />
- );
+ return null;
+ // <BaseStandardSection
+ // title={title}
+ // data={data}
+ // isFetching={isFetching}
+ // error={error}
+ // refetch={refetch}
+ // getKey={getKey}
+ // getLabel={getLabel}
+ // getValue={getValue}
+ // getLink={getLink}
+ // countLabel={countLabel}
+ // filterParameter={filterParameter}
+ // />
}
diff --git a/server/src/api/analytics/getSingleCol.ts b/server/src/api/analytics/getSingleCol.ts
index 7a509988c..e51816694 100644
--- a/server/src/api/analytics/getSingleCol.ts
+++ b/server/src/api/analytics/getSingleCol.ts
@@ -26,10 +26,13 @@ interface GenericRequest {
type GetSingleColResponse = {
value: string;
+ // count means sessions where this page was the entry/exit
count: number;
percentage: number;
- avg_session_duration?: number;
- bounce_rate?: number;
+
+ pageviews?: number; // pageviews of this specific page when it was an entry/exit
+ pageviews_percentage?: number;
+ time_on_page_seconds?: number; // avg time on this page when it was an entry/exit
}[];
const getQuery = (request: FastifyRequest<GenericRequest>) => {
@@ -38,6 +41,13 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
const site = request.params.site;
const filterStatement = getFilterStatement(filters);
+ const timeStatement = getTimeStatement(
+ minutes
+ ? { pastMinutes: minutes }
+ : {
+ date: { startDate, endDate, timezone },
+ }
+ );
const percentageStatement = `ROUND(
COUNT(distinct(session_id)) * 100.0 / SUM(COUNT(distinct(session_id))) OVER (),
@@ -56,13 +66,7 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
AND event_name IS NOT NULL
AND event_name <> ''
${filterStatement}
- ${getTimeStatement(
- minutes
- ? { pastMinutes: minutes }
- : {
- date: { startDate, endDate, timezone },
- }
- )}
+ ${timeStatement}
AND type = 'custom_event'
GROUP BY event_name ORDER BY count desc
${limit ? `LIMIT ${limit}` : ""};
@@ -70,35 +74,124 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
}
if (parameter === "exit_page" || parameter === "entry_page") {
- const arg = parameter === "exit_page" ? "argMax" : "argMin";
+ const isEntry = parameter === "entry_page";
+ const orderDirection = isEntry ? "ASC" : "DESC";
+ const rowNumFilter = isEntry ? "row_num = 1" : "row_num = 1"; // Need argMax logic instead if last row needed reliably
+
+ // For exit page, row_number() might not be the most robust if events aren't perfectly ordered.
+ // argMax(timestamp) per session might be better but makes getting the time_diff harder.
+ // Sticking with row_number for now, assuming reasonable ordering.
return `
- SELECT
+ WITH RelevantEvents AS (
+ -- Select all pageview events matching filters and time range
+ SELECT *
+ FROM events
+ WHERE
+ site_id = ${site}
+ AND type = 'pageview'
+ ${filterStatement}
+ ${timeStatement}
+ ),
+ EventTimes AS (
+ -- Calculate next timestamp within each session for duration calculation
+ SELECT
+ session_id,
+ pathname,
+ timestamp,
+ leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp,
+ -- Assign row number to identify first/last event per session
+ row_number() OVER (PARTITION BY session_id ORDER BY timestamp ${orderDirection}) as row_num
+ FROM RelevantEvents
+ ),
+ PageDurations AS (
+ -- Calculate duration for each pageview and keep row number
+ SELECT
+ session_id,
+ pathname,
+ timestamp,
+ next_timestamp,
+ row_num,
+ if(isNull(next_timestamp), 0, dateDiff('second', timestamp, next_timestamp)) as time_diff_seconds
+ FROM EventTimes
+ ),
+ FilteredDurations AS (
+ -- Filter durations to only include the entry or exit event
+ SELECT *
+ FROM PageDurations
+ WHERE ${rowNumFilter}
+ ),
+ PathStats AS (
+ -- Aggregate stats for the filtered entry/exit pages
+ SELECT
+ pathname,
+ -- Count distinct sessions where this path was entry/exit
+ count(DISTINCT session_id) as unique_sessions,
+ -- Count pageviews for this path when it was entry/exit (should be same as unique_sessions here)
+ count() as visits,
+ -- Calculate average time spent on this page when it was the entry/exit page
+ avg(if(time_diff_seconds < 0, 0, if(time_diff_seconds > 1800, 1800, time_diff_seconds))) as avg_time_on_page_seconds
+ FROM FilteredDurations
+ WHERE pathname IS NOT NULL AND pathname <> ''
+ GROUP BY pathname
+ )
+ -- Final selection with percentages
+ SELECT
pathname as value,
- COUNT(distinct(session_id)) as count,
- ${percentageStatement}
- FROM (
+ unique_sessions as count,
+ round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage,
+ visits as pageviews,
+ round((visits / sum(visits) OVER ()) * 100, 2) as pageviews_percentage,
+ avg_time_on_page_seconds as time_on_page_seconds
+ FROM PathStats
+ ORDER BY unique_sessions DESC
+ ${limit ? `LIMIT ${limit}` : ""};`;
+ }
+
+ if (parameter === "pathname") {
+ return `
+ WITH EventTimes AS (
SELECT
session_id,
- ${arg}(hostname, timestamp) AS hostname,
- ${arg}(pathname, timestamp) AS pathname
- FROM events
- WHERE
- site_id = ${site}
+ pathname,
+ timestamp,
+ leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp
+ FROM events
+ WHERE
+ site_id = ${site}
+ AND type = 'pageview'
${filterStatement}
- ${getTimeStatement(
- minutes
- ? { pastMinutes: minutes }
- : {
- date: { startDate, endDate, timezone },
- }
- )}
- // AND type = 'pageview'
- GROUP BY session_id
- ) AS query
- WHERE pathname IS NOT NULL AND pathname <> ''
- GROUP BY value ORDER BY count desc
- ${limit ? `LIMIT ${limit}` : ""};`;
+ ${timeStatement}
+ ),
+ PageDurations AS (
+ SELECT
+ session_id,
+ pathname,
+ timestamp,
+ next_timestamp,
+ if(isNull(next_timestamp), 0, dateDiff('second', timestamp, next_timestamp)) as time_diff_seconds
+ FROM EventTimes
+ ),
+ PathStats AS (
+ SELECT
+ pathname,
+ count() as visits,
+ count(DISTINCT session_id) as unique_sessions,
+ avg(if(time_diff_seconds < 0, 0, if(time_diff_seconds > 1800, 1800, time_diff_seconds))) as avg_time_on_page_seconds
+ FROM PageDurations
+ GROUP BY pathname
+ )
+ SELECT
+ pathname as value,
+ unique_sessions as count,
+ round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage,
+ visits as pageviews,
+ round((visits / sum(visits) OVER ()) * 100, 2) as pageviews_percentage,
+ avg_time_on_page_seconds as time_on_page_seconds
+ FROM PathStats
+ ORDER BY unique_sessions DESC
+ ${limit ? `LIMIT ${limit}` : ""};
+ `;
}
return `
@@ -112,37 +205,11 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
AND ${geSqlParam(parameter)} IS NOT NULL
AND ${geSqlParam(parameter)} <> ''
${filterStatement}
- ${getTimeStatement(
- minutes
- ? { pastMinutes: minutes }
- : {
- date: { startDate, endDate, timezone },
- }
- )}
+ ${timeStatement}
// AND type = 'pageview'
GROUP BY value ORDER BY count desc
${limit ? `LIMIT ${limit}` : ""};
`;
- // return `
- // SELECT
- // ${parameter} as value,
- // count() as count,
- // ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (), 2) as percentage,
- // ROUND(AVG(dateDiff('second', session_start, session_end))) as avg_session_duration,
- // ROUND(SUM(if(pageviews = 1, 1, 0)) * 100.0 / COUNT(), 2) as bounce_rate
- // FROM
- // sessions_mv
- // WHERE
- // site_id = ${site}
- // AND notEmpty(${parameter})
- // ${filterStatement}
- // ${getTimeStatement(startDate, endDate, timezone, "sessions")}
- // GROUP BY
- // ${parameter}
- // ORDER BY
- // COUNT() DESC
- // ${limit ? `LIMIT ${limit}` : ""};
- // `;
};
export async function getSingleCol(
@@ -169,6 +236,8 @@ export async function getSingleCol(
return res.send({ data });
} catch (error) {
console.error(`Error fetching ${parameter}:`, error);
+ // Add query to error log for easier debugging
+ console.error("Failed query:", query);
return res.status(500).send({ error: `Failed to fetch ${parameter}` });
}
}