Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

MySQL query optimization. Added loading component.

Mike Caoccb98f836fef
diff --git a/components/common/Loading.js b/components/common/Loading.js
new file mode 100644
index 000000000..46e72f178
--- /dev/null
+++ b/components/common/Loading.js
@@ -0,0 +1,13 @@
+import React from 'react';
+import classNames from 'classnames';
+import styles from './Loading.module.css';
+
+export default function Loading({ className }) {
+ return (
+ <div className={classNames(styles.loading, className)}>
+ <div />
+ <div />
+ <div />
+ </div>
+ );
+}
diff --git a/components/common/Loading.module.css b/components/common/Loading.module.css
new file mode 100644
index 000000000..2a210078b
--- /dev/null
+++ b/components/common/Loading.module.css
@@ -0,0 +1,41 @@
+@keyframes blink {
+ 0% {
+ opacity: 0.2;
+ }
+ 20% {
+ opacity: 1;
+ }
+ 100% {
+ opacity: 0.2;
+ }
+}
+
+.loading {
+ display: flex;
+ position: absolute;
+ top: 50%;
+ left: 50%;
+ transform: translate(-50%, -50%);
+ margin: 0;
+}
+
+.loading div {
+ width: 10px;
+ height: 10px;
+ border-radius: 100%;
+ background: var(--gray400);
+ animation: blink 1.4s infinite;
+ animation-fill-mode: both;
+}
+
+.loading div + div {
+ margin-left: 10px;
+}
+
+.loading div:nth-child(2) {
+ animation-delay: 0.2s;
+}
+
+.loading div:nth-child(3) {
+ animation-delay: 0.4s;
+}
diff --git a/components/layout/MenuLayout.module.css b/components/layout/MenuLayout.module.css
index 62233fd6c..1bdd74739 100644
--- a/components/layout/MenuLayout.module.css
+++ b/components/layout/MenuLayout.module.css
@@ -10,6 +10,7 @@
}
.container .content {
+ position: relative;
border-left: 1px solid var(--gray300);
padding-left: 30px;
}
diff --git a/components/metrics/BarChart.js b/components/metrics/BarChart.js
index fdce63308..25c14fe5a 100644
--- a/components/metrics/BarChart.js
+++ b/components/metrics/BarChart.js
@@ -120,6 +120,7 @@ export default function BarChart({
options.scales.xAxes[0].time.unit = unit;
options.scales.xAxes[0].ticks.callback = renderLabel;
+ options.animation.duration = animationDuration;
onUpdate(chart.current);
};
@@ -133,7 +134,7 @@ export default function BarChart({
updateChart();
}
}
- }, [datasets]);
+ }, [datasets, unit, animationDuration]);
return (
<>
diff --git a/components/metrics/EventsChart.js b/components/metrics/EventsChart.js
index 1825116fd..7a751b5b6 100644
--- a/components/metrics/EventsChart.js
+++ b/components/metrics/EventsChart.js
@@ -42,7 +42,7 @@ export default function EventsChart({ websiteId, startDate, endDate, unit }) {
unit,
tz: getTimezone(),
});
- console.log({ data });
+
const map = data.reduce((obj, { x, t, y }) => {
if (!obj[x]) {
obj[x] = [];
diff --git a/components/metrics/MetricsTable.js b/components/metrics/MetricsTable.js
index bd66cb85a..566ceed37 100644
--- a/components/metrics/MetricsTable.js
+++ b/components/metrics/MetricsTable.js
@@ -8,6 +8,7 @@ import { get } from 'lib/web';
import { percentFilter } from 'lib/filters';
import { formatNumber, formatLongNumber } from 'lib/format';
import styles from './MetricsTable.module.css';
+import Loading from '../common/Loading';
export default function MetricsTable({
title,
@@ -79,35 +80,37 @@ export default function MetricsTable({
}
}, [websiteId, startDate, endDate, type]);
- if (!data) {
- return null;
- }
-
return (
<div className={classNames(styles.container, className)}>
- <div className={styles.header}>
- <div className={styles.title}>{title}</div>
- {headerComponent}
- <div className={styles.metric} onClick={handleSetFormat}>
- {metric}
- </div>
- </div>
- <div className={styles.body}>
- {limit
- ? rankings.map(row => getRow(row))
- : data?.length > 0 && (
- <FixedSizeList height={500} itemCount={rankings.length} itemSize={30}>
- {Row}
- </FixedSizeList>
+ {data ? (
+ <>
+ <div className={styles.header}>
+ <div className={styles.title}>{title}</div>
+ {headerComponent}
+ <div className={styles.metric} onClick={handleSetFormat}>
+ {metric}
+ </div>
+ </div>
+ <div className={styles.body}>
+ {limit
+ ? rankings.map(row => getRow(row))
+ : data?.length > 0 && (
+ <FixedSizeList height={500} itemCount={rankings.length} itemSize={30}>
+ {Row}
+ </FixedSizeList>
+ )}
+ </div>
+ <div className={styles.footer}>
+ {limit && data.length > limit && (
+ <Button icon={<Arrow />} size="xsmall" onClick={() => onExpand(type)}>
+ <div>More</div>
+ </Button>
)}
- </div>
- <div className={styles.footer}>
- {limit && data.length > limit && (
- <Button icon={<Arrow />} size="xsmall" onClick={() => onExpand(type)}>
- <div>More</div>
- </Button>
- )}
- </div>
+ </div>
+ </>
+ ) : (
+ <Loading />
+ )}
</div>
);
}
diff --git a/components/metrics/PageviewsChart.js b/components/metrics/PageviewsChart.js
index 1052452b2..f8aa0cdcb 100644
--- a/components/metrics/PageviewsChart.js
+++ b/components/metrics/PageviewsChart.js
@@ -1,16 +1,15 @@
import React from 'react';
+import CheckVisible from 'components/helpers/CheckVisible';
import BarChart from './BarChart';
-export default function PageviewsChart({ websiteId, data, unit, className, animationDuration }) {
+export default function PageviewsChart({ websiteId, data, unit, className }) {
const handleUpdate = chart => {
const {
data: { datasets },
- options,
} = chart;
datasets[0].data = data.uniques;
datasets[1].data = data.pageviews;
- options.animation.duration = animationDuration;
chart.update();
};
@@ -20,30 +19,35 @@ export default function PageviewsChart({ websiteId, data, unit, className, anima
}
return (
- <BarChart
- className={className}
- chartId={websiteId}
- datasets={[
- {
- label: 'unique visitors',
- data: data.uniques,
- lineTension: 0,
- backgroundColor: 'rgb(38, 128, 235, 0.4)',
- borderColor: 'rgb(13, 102, 208, 0.4)',
- borderWidth: 1,
- },
- {
- label: 'page views',
- data: data.pageviews,
- lineTension: 0,
- backgroundColor: 'rgb(38, 128, 235, 0.2)',
- borderColor: 'rgb(13, 102, 208, 0.2)',
- borderWidth: 1,
- },
- ]}
- unit={unit}
- records={data.pageviews.length}
- onUpdate={handleUpdate}
- />
+ <CheckVisible>
+ {visible => (
+ <BarChart
+ className={className}
+ chartId={websiteId}
+ datasets={[
+ {
+ label: 'unique visitors',
+ data: data.uniques,
+ lineTension: 0,
+ backgroundColor: 'rgb(38, 128, 235, 0.4)',
+ borderColor: 'rgb(13, 102, 208, 0.4)',
+ borderWidth: 1,
+ },
+ {
+ label: 'page views',
+ data: data.pageviews,
+ lineTension: 0,
+ backgroundColor: 'rgb(38, 128, 235, 0.2)',
+ borderColor: 'rgb(13, 102, 208, 0.2)',
+ borderWidth: 1,
+ },
+ ]}
+ unit={unit}
+ records={data.pageviews.length}
+ animationDuration={visible ? 300 : 0}
+ onUpdate={handleUpdate}
+ />
+ )}
+ </CheckVisible>
);
}
diff --git a/components/metrics/WebsiteChart.js b/components/metrics/WebsiteChart.js
index f93a985a3..9c53a5603 100644
--- a/components/metrics/WebsiteChart.js
+++ b/components/metrics/WebsiteChart.js
@@ -1,7 +1,6 @@
-import React, { useState, useEffect, useMemo, useRef } from 'react';
+import React, { useState, useEffect, useMemo } from 'react';
import classNames from 'classnames';
import PageviewsChart from './PageviewsChart';
-import CheckVisible from '../helpers/CheckVisible';
import MetricsBar from './MetricsBar';
import QuickButtons from './QuickButtons';
import DateFilter from '../common/DateFilter';
@@ -74,19 +73,10 @@ export default function WebsiteChart({
</StickyHeader>
</div>
<div className="row">
- <CheckVisible className="col">
- {visible => (
- <>
- <PageviewsChart
- websiteId={websiteId}
- data={{ pageviews, uniques }}
- unit={unit}
- animationDuration={visible ? 300 : 0}
- />
- <QuickButtons value={value} onChange={handleDateChange} />
- </>
- )}
- </CheckVisible>
+ <div className="col">
+ <PageviewsChart websiteId={websiteId} data={{ pageviews, uniques }} unit={unit} />
+ <QuickButtons value={value} onChange={handleDateChange} />
+ </div>
</div>
</>
);
diff --git a/lib/date.js b/lib/date.js
index 6e10f6108..80980e271 100644
--- a/lib/date.js
+++ b/lib/date.js
@@ -98,6 +98,12 @@ export function getDateArray(data, startDate, endDate, unit) {
function findData(t) {
const x = data.find(e => {
+ console.log(
+ new Date(e.t),
+ getLocalTime(new Date(e.t)),
+ getLocalTime(new Date(e.t)).getTime(),
+ normalize(new Date(t)).getTime(),
+ );
return getLocalTime(new Date(e.t)).getTime() === normalize(new Date(t)).getTime();
});
diff --git a/lib/filters.js b/lib/filters.js
index a1bf1b96f..09f27e70c 100644
--- a/lib/filters.js
+++ b/lib/filters.js
@@ -49,12 +49,13 @@ export const urlFilter = (data, { domain, raw }) => {
};
export const refFilter = (data, { domain, domainOnly, raw }) => {
+ const regex = new RegExp(domain.startsWith('http') ? domain : `http[s]?://${domain}`);
+
const isValidRef = ref => {
return ref !== '' && !ref.startsWith('/') && !ref.startsWith('#');
};
if (raw) {
- const regex = new RegExp(`http[s]?://${domain}`);
return data.filter(({ x }) => isValidRef(x) && !regex.test(x));
}
@@ -62,7 +63,7 @@ export const refFilter = (data, { domain, domainOnly, raw }) => {
try {
const { hostname, origin, pathname, searchParams, protocol } = new URL(url);
- if (hostname === domain) {
+ if (hostname === domain || regex.test(url)) {
return null;
}
diff --git a/lib/queries.js b/lib/queries.js
index 1907a9549..7c8a9875e 100644
--- a/lib/queries.js
+++ b/lib/queries.js
@@ -5,10 +5,28 @@ import { subMinutes } from 'date-fns';
const POSTGRESQL = 'postgresql';
const MYSQL = 'mysql';
+const DATE_FORMATS = {
+ minute: '%Y-%m-%d %H:%i:00',
+ hour: '%Y-%m-%d %H:00:00',
+ day: '%Y-%m-%d',
+ month: '%Y-%m-01',
+ year: '%Y-01-01',
+};
+
export function getDatabase() {
return process.env.DATABASE_TYPE || process.env.DATABASE_URL.split(':')[0];
}
+export function getDateQuery(field, unit, timezone) {
+ if (timezone) {
+ const tz = moment.tz(timezone).format('Z');
+
+ return `DATE_FORMAT(convert_tz(${field},'+00:00','${tz}'), '${DATE_FORMATS[unit]}')`;
+ }
+
+ return `DATE_FORMAT(${field}, '${DATE_FORMATS[unit]}')`;
+}
+
export async function getWebsiteById(website_id) {
return runQuery(
prisma.website.findOne({
@@ -264,7 +282,7 @@ export function getMetrics(website_id, start_at, end_at) {
sum(t.time) as "totaltime"
from (
select session_id,
- date_trunc('hour', created_at),
+ ${getDateQuery('created_at', 'hour')},
count(*) c,
floor(unix_timestamp(max(created_at)) - unix_timestamp(min(created_at))) as "time"
from pageview
@@ -296,7 +314,7 @@ export function getPageviews(
return prisma.$queryRaw(
`
select date_trunc('${unit}', created_at at time zone '${timezone}') t,
- count(${count}) y
+ count(${count}) y
from pageview
where website_id=$1
and created_at between $2 and $3
@@ -310,11 +328,10 @@ export function getPageviews(
}
if (db === MYSQL) {
- const tz = moment.tz(timezone).format('Z');
return prisma.$queryRaw(
`
- select date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t,
- count(${count}) y
+ select ${getDateQuery('created_at', unit, timezone)} t,
+ count(${count}) y
from pageview
where website_id=?
and created_at between ? and ?
@@ -424,12 +441,11 @@ export function getEvents(website_id, start_at, end_at, timezone = 'utc', unit =
}
if (db === MYSQL) {
- const tz = moment.tz(timezone).format('Z');
return prisma.$queryRaw(
`
select
event_value x,
- date_trunc('${unit}', convert_tz(created_at,'+00:00','${tz}')) t,
+ ${getDateQuery('created_at', unit, timezone)} t,
count(*) y
from event
where website_id=?
diff --git a/sql/schema.mysql.sql b/sql/schema.mysql.sql
index 106a6748b..a4fd87f05 100644
--- a/sql/schema.mysql.sql
+++ b/sql/schema.mysql.sql
@@ -3,7 +3,6 @@ drop table if exists pageview;
drop table if exists session;
drop table if exists website;
drop table if exists account;
-drop function if exists date_trunc;
create table account (
user_id int unsigned not null auto_increment primary key,
@@ -72,37 +71,10 @@ create index pageview_created_at_idx on pageview(created_at);
create index pageview_website_id_idx on pageview(website_id);
create index pageview_session_id_idx on pageview(session_id);
create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
+create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
create index event_created_at_idx on event(created_at);
create index event_website_id_idx on event(website_id);
create index event_session_id_idx on event(session_id);
-delimiter $$
-
-create function date_trunc(
- in_granularity enum('minute', 'hour', 'day', 'month', 'year'),
- in_datetime datetime(6)
-)
-returns datetime(6)
-deterministic
-begin
- if (in_granularity = 'minute') then
- return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:%i:00.0000');
- end if;
- if (in_granularity = 'hour') then
- return DATE_FORMAT(in_datetime, '%Y-%m-%d %H:00:00.0000');
- end if;
- if (in_granularity = 'day') then
- return DATE_FORMAT(in_datetime, '%Y-%m-%d 00:00:00.0000');
- end if;
- if (in_granularity = 'month') then
- return DATE_FORMAT(in_datetime, '%Y-%m-01 00:00:00.0000');
- end if;
- if (in_granularity = 'year') then
- return DATE_FORMAT(in_datetime, '%Y-01-01 00:00:00.0000');
- end if;
-end;
-
-$$
-
insert into account (username, password, is_admin) values ('admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa', true);
\ No newline at end of file
diff --git a/sql/schema.postgresql.sql b/sql/schema.postgresql.sql
index 435333496..28297c4a8 100644
--- a/sql/schema.postgresql.sql
+++ b/sql/schema.postgresql.sql
@@ -65,6 +65,7 @@ create index pageview_created_at_idx on pageview(created_at);
create index pageview_website_id_idx on pageview(website_id);
create index pageview_session_id_idx on pageview(session_id);
create index pageview_website_id_created_at_idx on pageview(website_id, created_at);
+create index pageview_website_id_session_id_created_at_idx on pageview(website_id, session_id, created_at);
create index event_created_at_idx on event(created_at);
create index event_website_id_idx on event(website_id);