Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

only use hourly table, remove daily table logic, fix updatechart undefined

Francis Cao174b9e437637
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
index 9082884b2..29cdeabc2 100644
--- a/db/clickhouse/schema.sql
+++ b/db/clickhouse/schema.sql
@@ -78,6 +78,8 @@ CREATE TABLE umami.website_event_stats_hourly
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
+ screen LowCardinality(String),
+ language LowCardinality(String),
country LowCardinality(String),
subdivision1 LowCardinality(String),
city String,
@@ -103,8 +105,7 @@ ORDER BY (
visit_id,
event_type
)
-SAMPLE BY cityHash64(visit_id)
-TTL created_at + INTERVAL 10 DAY;
+SAMPLE BY cityHash64(visit_id);
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
@@ -117,6 +118,8 @@ SELECT
browser,
os,
device,
+ screen,
+ language,
country,
subdivision1,
city,
@@ -140,6 +143,8 @@ FROM (SELECT
browser,
os,
device,
+ screen,
+ language,
country,
subdivision1,
city,
@@ -163,106 +168,8 @@ GROUP BY website_id,
browser,
os,
device,
- country,
- subdivision1,
- city,
- event_type,
- timestamp);
-
--- stats daily
-CREATE TABLE umami.website_event_stats_daily
-(
- website_id UUID,
- session_id UUID,
- visit_id UUID,
- hostname LowCardinality(String),
- browser LowCardinality(String),
- os LowCardinality(String),
- device LowCardinality(String),
- country LowCardinality(String),
- subdivision1 LowCardinality(String),
- city String,
- entry_url AggregateFunction(argMin, String, DateTime('UTC')),
- exit_url AggregateFunction(argMax, String, DateTime('UTC')),
- url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
- url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
- referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
- page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
- event_type UInt32,
- event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
- views SimpleAggregateFunction(sum, UInt64),
- min_time SimpleAggregateFunction(min, DateTime('UTC')),
- max_time SimpleAggregateFunction(max, DateTime('UTC')),
- created_at Datetime('UTC')
-)
-ENGINE = AggregatingMergeTree
-PARTITION BY toYYYYMM(created_at)
-ORDER BY (
- website_id,
- toStartOfDay(created_at),
- cityHash64(visit_id),
- visit_id,
- event_type
-)
-SAMPLE BY cityHash64(visit_id);
-
-CREATE MATERIALIZED VIEW umami.website_event_stats_daily_mv
-TO umami.website_event_stats_daily
-AS
-SELECT
- website_id,
- session_id,
- visit_id,
- hostname,
- browser,
- os,
- device,
- country,
- subdivision1,
- city,
- entry_url,
- exit_url,
- url_paths as url_path,
- url_query,
- referrer_domain,
- page_title,
- event_type,
- event_name,
- views,
- min_time,
- max_time,
- timestamp as created_at
-FROM (SELECT
- website_id,
- session_id,
- visit_id,
- hostname,
- browser,
- os,
- device,
- country,
- subdivision1,
- city,
- argMinState(url_path, created_at) entry_url,
- argMaxState(url_path, created_at) exit_url,
- arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
- arrayFilter(x -> x != '', groupArray(url_query)) url_query,
- arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
- arrayFilter(x -> x != '', groupArray(page_title)) page_title,
- event_type,
- if(event_type = 2, groupArray(event_name), []) event_name,
- sumIf(1, event_type = 1) views,
- min(created_at) min_time,
- max(created_at) max_time,
- toStartOfDay(created_at) timestamp
-FROM umami.website_event
-GROUP BY website_id,
- session_id,
- visit_id,
- hostname,
- browser,
- os,
- device,
+ screen,
+ language,
country,
subdivision1,
city,
diff --git a/src/components/charts/Chart.tsx b/src/components/charts/Chart.tsx
index 926defaf9..a4badbcef 100644
--- a/src/components/charts/Chart.tsx
+++ b/src/components/charts/Chart.tsx
@@ -79,18 +79,20 @@ export function Chart({
};
const updateChart = (data: any) => {
- if ((data?.datasets?.length || 0) === chart.current.data.datasets.length) {
- chart.current.data.datasets.forEach((dataset: { data: any }, index: string | number) => {
- if (data?.datasets[index]) {
- dataset.data = data?.datasets[index]?.data;
-
- if (chart.current.legend.legendItems[index]) {
- chart.current.legend.legendItems[index].text = data?.datasets[index]?.label;
+ if (data.datasets) {
+ if (data.datasets.length === chart.current.data.datasets.length) {
+ chart.current.data.datasets.forEach((dataset: { data: any }, index: string | number) => {
+ if (data?.datasets[index]) {
+ dataset.data = data?.datasets[index]?.data;
+
+ if (chart.current.legend.legendItems[index]) {
+ chart.current.legend.legendItems[index].text = data?.datasets[index]?.label;
+ }
}
- }
- });
- } else {
- chart.current.data.datasets = data.datasets;
+ });
+ } else {
+ chart.current.data.datasets = data.datasets;
+ }
}
chart.current.options = options;
diff --git a/src/pages/api/websites/[websiteId]/metrics.ts b/src/pages/api/websites/[websiteId]/metrics.ts
index b37c38f74..3dac217b2 100644
--- a/src/pages/api/websites/[websiteId]/metrics.ts
+++ b/src/pages/api/websites/[websiteId]/metrics.ts
@@ -64,7 +64,7 @@ export default async (
await useAuth(req, res);
await useValidate(schema, req, res);
- const { websiteId, type, limit, offset, search, unit } = req.query;
+ const { websiteId, type, limit, offset, search } = req.query;
if (req.method === 'GET') {
if (!(await canViewWebsite(req.auth, websiteId))) {
@@ -89,7 +89,7 @@ export default async (
}
if (SESSION_COLUMNS.includes(type)) {
- const data = await getSessionMetrics(websiteId, type, filters, limit, offset, unit as string);
+ const data = await getSessionMetrics(websiteId, type, filters, limit, offset);
if (type === 'language') {
const combined = {};
@@ -111,14 +111,7 @@ export default async (
}
if (EVENT_COLUMNS.includes(type)) {
- const data = await getPageviewMetrics(
- websiteId,
- type,
- filters,
- limit,
- offset,
- unit as string,
- );
+ const data = await getPageviewMetrics(websiteId, type, filters, limit, offset);
return ok(res, data);
}
diff --git a/src/pages/api/websites/[websiteId]/stats.ts b/src/pages/api/websites/[websiteId]/stats.ts
index 1c684dbed..9ca84c744 100644
--- a/src/pages/api/websites/[websiteId]/stats.ts
+++ b/src/pages/api/websites/[websiteId]/stats.ts
@@ -56,7 +56,7 @@ export default async (
await useAuth(req, res);
await useValidate(schema, req, res);
- const { websiteId, compare, unit } = req.query;
+ const { websiteId, compare } = req.query;
if (req.method === 'GET') {
if (!(await canViewWebsite(req.auth, websiteId))) {
@@ -72,13 +72,13 @@ export default async (
const filters = getRequestFilters(req);
- const metrics = await getWebsiteStats(websiteId, unit as string, {
+ const metrics = await getWebsiteStats(websiteId, {
...filters,
startDate,
endDate,
});
- const prevPeriod = await getWebsiteStats(websiteId, unit as string, {
+ const prevPeriod = await getWebsiteStats(websiteId, {
...filters,
startDate: compareStartDate,
endDate: compareEndDate,
diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.ts
index c97c11ad2..362321358 100644
--- a/src/queries/analytics/events/getEventMetrics.ts
+++ b/src/queries/analytics/events/getEventMetrics.ts
@@ -51,8 +51,6 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.customEvent,
});
- const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
-
return rawQuery(
`
select
@@ -62,7 +60,7 @@ async function clickhouseQuery(
from (
select arrayJoin(event_name) as event_name,
created_at
- from ${table} website_event
+ from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts
index 125715ca6..ebe711a03 100644
--- a/src/queries/analytics/getWebsiteStats.ts
+++ b/src/queries/analytics/getWebsiteStats.ts
@@ -6,7 +6,7 @@ import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
export async function getWebsiteStats(
- ...args: [websiteId: string, unit: string, filters: QueryFilters]
+ ...args: [websiteId: string, filters: QueryFilters]
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
> {
@@ -18,7 +18,6 @@ export async function getWebsiteStats(
async function relationalQuery(
websiteId: string,
- unit: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
@@ -59,7 +58,6 @@ async function relationalQuery(
async function clickhouseQuery(
websiteId: string,
- unit: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
@@ -69,7 +67,6 @@ async function clickhouseQuery(
...filters,
eventType: EVENT_TYPE.pageView,
});
- const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@@ -79,7 +76,7 @@ async function clickhouseQuery(
uniq(visit_id) as "visits",
sumIf(1, views = 1) as "bounces",
sum(max_time-min_time) as "totaltime"
- from ${table} "website_event"
+ from website_event_stats_hourly "website_event"
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts
index d66ec6aa7..ccfe4ef03 100644
--- a/src/queries/analytics/pageviews/getPageviewMetrics.ts
+++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts
@@ -1,4 +1,3 @@
-/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
import clickhouse from 'lib/clickhouse';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
@@ -6,14 +5,7 @@ import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
export async function getPageviewMetrics(
- ...args: [
- websiteId: string,
- type: string,
- filters: QueryFilters,
- limit?: number,
- offset?: number,
- unit?: string,
- ]
+ ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@@ -27,7 +19,6 @@ async function relationalQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
- unit: string,
) {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = prisma;
@@ -91,7 +82,6 @@ async function clickhouseQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
- unit: string,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = clickhouse;
@@ -121,15 +111,13 @@ async function clickhouseQuery(
groupByQuery = 'group by x';
}
- const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
-
return rawQuery(
`
select g.t as x,
count(*) as y
from (
select ${columnQuery} as t
- from ${table} website_event
+ from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts
index 0bb16ca9b..8d757de31 100644
--- a/src/queries/analytics/pageviews/getPageviewStats.ts
+++ b/src/queries/analytics/pageviews/getPageviewStats.ts
@@ -46,7 +46,6 @@ async function clickhouseQuery(
...filters,
eventType: EVENT_TYPE.pageView,
});
- const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@@ -57,7 +56,7 @@ async function clickhouseQuery(
select
${getDateSQL('created_at', unit, timezone)} as t,
sum(views) as y
- from ${table} website_event
+ from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts
index 93e36a55f..9baf2a5c0 100644
--- a/src/queries/analytics/sessions/getSessionMetrics.ts
+++ b/src/queries/analytics/sessions/getSessionMetrics.ts
@@ -1,19 +1,11 @@
-/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
-import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
export async function getSessionMetrics(
- ...args: [
- websiteId: string,
- type: string,
- filters: QueryFilters,
- limit?: number,
- offset?: number,
- unit?: string,
- ]
+ ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@@ -27,7 +19,6 @@ async function relationalQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
- unit: string,
) {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = prisma;
@@ -71,7 +62,6 @@ async function clickhouseQuery(
filters: QueryFilters,
limit: number = 500,
offset: number = 0,
- unit: string,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse;
@@ -80,7 +70,6 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.pageView,
});
const includeCountry = column === 'city' || column === 'subdivision1';
- const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@@ -88,7 +77,7 @@ async function clickhouseQuery(
${column} x,
uniq(session_id) y
${includeCountry ? ', country' : ''}
- from ${table} website_event
+ from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts
index 2b57b922b..0fe0b8a55 100644
--- a/src/queries/analytics/sessions/getSessionStats.ts
+++ b/src/queries/analytics/sessions/getSessionStats.ts
@@ -46,7 +46,6 @@ async function clickhouseQuery(
...filters,
eventType: EVENT_TYPE.pageView,
});
- const table = unit === 'hour' ? 'website_event_stats_hourly' : 'website_event_stats_daily';
return rawQuery(
`
@@ -57,7 +56,7 @@ async function clickhouseQuery(
select
${getDateSQL('created_at', unit, timezone)} as t,
uniq(session_id) as y
- from ${table} website_event
+ from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}