Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
add subdivision1/2, cities to query logic
Francis Cao55a586fe27a9
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sqlindex 15a9c33eb..3a04aea77 100644--- a/db/clickhouse/schema.sql+++ b/db/clickhouse/schema.sql@@ -15,7 +15,7 @@ CREATE TABLE eventscreen LowCardinality(String),language LowCardinality(String),country LowCardinality(String),- subdivision LowCardinality(String),+ subdivision1 LowCardinality(String),subdivision2 LowCardinality(String),city String,--pageviewdiff --git a/lib/clickhouse.ts b/lib/clickhouse.tsindex f59e04c67..5420e969c 100644--- a/lib/clickhouse.ts+++ b/lib/clickhouse.ts@@ -115,6 +115,9 @@ function getFilterQuery(filters = {}, params = {}) {case 'os':case 'browser':case 'device':+ case 'subdivision1':+ case 'subdivision2':+ case 'city':case 'country':arr.push(`and ${key} = {${key}:String}`);params[key] = filter;@@ -147,11 +150,24 @@ function getFilterQuery(filters = {}, params = {}) {}function parseFilters(filters: any = {}, params: any = {}) {- const { domain, url, eventUrl, referrer, os, browser, device, country, eventName, query } =- filters;+ const {+ domain,+ url,+ eventUrl,+ referrer,+ os,+ browser,+ device,+ country,+ subdivision1,+ subdivision2,+ city,+ eventName,+ query,+ } = filters;const pageviewFilters = { domain, url, referrer, query };- const sessionFilters = { os, browser, device, country };+ const sessionFilters = { os, browser, device, country, subdivision1, subdivision2, city };const eventFilters = { url: eventUrl, eventName };return {diff --git a/lib/detect.js b/lib/detect.jsindex 74d95daac..508f6f3c4 100644--- a/lib/detect.js+++ b/lib/detect.js@@ -68,17 +68,17 @@ export async function getLocation(ip) {const result = lookup.get(ip);const country = result?.country?.iso_code ?? result?.registered_country?.iso_code;- const subdivision1 = result?.subdivisions[0].iso_code;- const subdivision2 = result?.subdivisions[1].iso_code;+ const subdivision1 = result?.subdivisions[0]?.iso_code;+ const subdivision2 = result?.subdivisions[1]?.iso_code;const city = result?.city?.names?.en;return { country, subdivision1, subdivision2, city };}export async function getClientInfo(req, { screen }) {- const location = await getLocation(ip);const userAgent = req.headers['user-agent'];const ip = getIpAddress(req);+ const location = await getLocation(ip);const country = location.country;const subdivision1 = location.subdivision1;const subdivision2 = location.subdivision2;diff --git a/lib/prisma.ts b/lib/prisma.tsindex 5136958cf..11db678bc 100644--- a/lib/prisma.ts+++ b/lib/prisma.ts@@ -135,6 +135,9 @@ function getFilterQuery(filters = {}, params = []): string {case 'os':case 'browser':case 'device':+ case 'subdivision1':+ case 'subdivision2':+ case 'city':case 'country':arr.push(`and ${key}=$${params.length + 1}`);params.push(decodeURIComponent(filter));@@ -171,11 +174,24 @@ function parseFilters(params = [],sessionKey = 'session_id',) {- const { domain, url, eventUrl, referrer, os, browser, device, country, eventName, query } =- filters;+ const {+ domain,+ url,+ eventUrl,+ referrer,+ os,+ browser,+ device,+ country,+ subdivision1,+ subdivision2,+ city,+ eventName,+ query,+ } = filters;const pageviewFilters = { domain, url, referrer, query };- const sessionFilters = { os, browser, device, country };+ const sessionFilters = { os, browser, device, country, subdivision1, subdivision2, city };const eventFilters = { url: eventUrl, eventName };return {@@ -184,7 +200,7 @@ function parseFilters(eventFilters,event: { eventName },joinSession:- os || browser || device || country+ os || browser || device || country || subdivision1 || subdivision2 || city? `inner join session on website_event.${sessionKey} = session.${sessionKey}`: '',filterQuery: getFilterQuery(filters, params),diff --git a/lib/session.js b/lib/session.jsindex f66984808..f8b35a769 100644--- a/lib/session.js+++ b/lib/session.js@@ -44,7 +44,8 @@ export async function findSession(req) {throw new Error(`Website not found: ${websiteId}`);}- const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload);+ const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =+ await getClientInfo(req, payload);const sessionId = uuid(websiteId, hostname, ip, userAgent);// Clickhouse does not require session lookup@@ -59,6 +60,9 @@ export async function findSession(req) {screen,language,country,+ subdivision1,+ subdivision2,+ city,};}@@ -84,6 +88,9 @@ export async function findSession(req) {screen,language,country,+ subdivision1,+ subdivision2,+ city,});} catch (e) {if (!e.message.toLowerCase().includes('unique constraint')) {diff --git a/pages/api/collect.ts b/pages/api/collect.tsindex 63bbdf1bc..24dcbee14 100644--- a/pages/api/collect.ts+++ b/pages/api/collect.ts@@ -19,6 +19,9 @@ export interface NextApiRequestCollect extends NextApiRequest {screen: string;language: string;country: string;+ subdivision1: string;+ subdivision2: string;+ city: string;};}diff --git a/pages/api/websites/[id]/metrics.ts b/pages/api/websites/[id]/metrics.tsindex 943580766..70dcd7a6f 100644--- a/pages/api/websites/[id]/metrics.ts+++ b/pages/api/websites/[id]/metrics.ts@@ -46,6 +46,9 @@ export interface WebsiteMetricsRequestQuery {browser: string;device: string;country: string;+ subdivision1: string;+ subdivision2: string;+ city: string;}export default async (@@ -66,6 +69,9 @@ export default async (browser,device,country,+ subdivision1,+ subdivision2,+ city,} = req.query;if (req.method === 'GET') {@@ -86,6 +92,9 @@ export default async (browser,device,country,+ subdivision1,+ subdivision2,+ city,},});@@ -131,6 +140,9 @@ export default async (browser: type !== 'browser' ? browser : undefined,device: type !== 'device' ? device : undefined,country: type !== 'country' ? country : undefined,+ subdivision1: type !== 'subdivision1' ? subdivision1 : undefined,+ subdivision2: type !== 'subdivision2' ? subdivision2 : undefined,+ city: type !== 'city' ? city : undefined,eventUrl: type !== 'url' && table === 'event' ? url : undefined,query: type === 'query' && table !== 'event' ? true : undefined,};diff --git a/pages/api/websites/[id]/pageviews.ts b/pages/api/websites/[id]/pageviews.tsindex 3264cb290..f21760496 100644--- a/pages/api/websites/[id]/pageviews.ts+++ b/pages/api/websites/[id]/pageviews.ts@@ -21,6 +21,9 @@ export interface WebsitePageviewRequestQuery {browser?: string;device?: string;country?: string;+ subdivision1?: string;+ subdivision2?: string;+ city?: string;}export default async (@@ -42,6 +45,9 @@ export default async (browser,device,country,+ subdivision1,+ subdivision2,+ city,} = req.query;if (req.method === 'GET') {@@ -70,6 +76,9 @@ export default async (browser,device,country,+ subdivision1,+ subdivision2,+ city,},}),getPageviewStats(websiteId, {@@ -84,6 +93,9 @@ export default async (browser,device,country,+ subdivision1,+ subdivision2,+ city,},}),]);diff --git a/pages/api/websites/[id]/stats.ts b/pages/api/websites/[id]/stats.tsindex 272626151..138ede608 100644--- a/pages/api/websites/[id]/stats.ts+++ b/pages/api/websites/[id]/stats.ts@@ -17,6 +17,9 @@ export interface WebsiteStatsRequestQuery {browser: string;device: string;country: string;+ subdivision1: string;+ subdivision2: string;+ city: string;}export default async (@@ -26,7 +29,20 @@ export default async (await useCors(req, res);await useAuth(req, res);- const { id: websiteId, startAt, endAt, url, referrer, os, browser, device, country } = req.query;+ const {+ id: websiteId,+ startAt,+ endAt,+ url,+ referrer,+ os,+ browser,+ device,+ country,+ subdivision1,+ subdivision2,+ city,+ } = req.query;if (req.method === 'GET') {if (!(await canViewWebsite(req.auth, websiteId))) {@@ -50,6 +66,9 @@ export default async (browser,device,country,+ subdivision1,+ subdivision2,+ city,},});const prevPeriod = await getWebsiteStats(websiteId, {@@ -62,6 +81,9 @@ export default async (browser,device,country,+ subdivision1,+ subdivision2,+ city,},});diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.tsindex 430ffd63b..81cb10b48 100644--- a/queries/analytics/event/saveEvent.ts+++ b/queries/analytics/event/saveEvent.ts@@ -19,6 +19,9 @@ export async function saveEvent(args: {screen?: string;language?: string;country?: string;+ subdivision1?: string;+ subdivision2?: string;+ city?: string;}) {return runQuery({[PRISMA]: () => relationalQuery(args),@@ -36,38 +39,33 @@ async function relationalQuery(data: {}) {const { websiteId, id: sessionId, url, eventName, eventData, referrer } = data;- const params = {- id: uuid(),- websiteId,- sessionId,- url: url?.substring(0, URL_LENGTH),- referrer: referrer?.substring(0, URL_LENGTH),- eventType: EVENT_TYPE.customEvent,- eventName: eventName?.substring(0, EVENT_NAME_LENGTH),- eventData,- };-return prisma.client.websiteEvent.create({- data: params,+ data: {+ id: uuid(),+ websiteId,+ sessionId,+ url: url?.substring(0, URL_LENGTH),+ referrer: referrer?.substring(0, URL_LENGTH),+ eventType: EVENT_TYPE.customEvent,+ eventName: eventName?.substring(0, EVENT_NAME_LENGTH),+ eventData,+ },});}-async function clickhouseQuery(data: {- id: string;- websiteId: string;- url: string;- referrer?: string;- eventName?: string;- eventData?: any;- hostname?: string;- browser?: string;- os?: string;- device?: string;- screen?: string;- language?: string;- country?: string;-}) {- const { websiteId, id: sessionId, url, eventName, eventData, country, ...args } = data;+async function clickhouseQuery(data) {+ const {+ websiteId,+ id: sessionId,+ url,+ eventName,+ eventData,+ country,+ subdivision1,+ subdivision2,+ city,+ ...args+ } = data;const { getDateFormat, sendMessage } = kafka;const website = await cache.fetchWebsite(websiteId);@@ -75,13 +73,16 @@ async function clickhouseQuery(data: {website_id: websiteId,session_id: sessionId,event_id: uuid(),+ rev_id: website?.revId || 0,+ country: country ? country : null,+ subdivision1: subdivision1 ? subdivision1 : null,+ subdivision2: subdivision2 ? subdivision2 : null,+ city: city ? city : null,url: url?.substring(0, URL_LENGTH),event_type: EVENT_TYPE.customEvent,event_name: eventName?.substring(0, EVENT_NAME_LENGTH),event_data: eventData ? JSON.stringify(eventData) : null,- rev_id: website?.revId || 0,created_at: getDateFormat(new Date()),- country: country ? country : null,...args,};diff --git a/queries/analytics/pageview/savePageView.ts b/queries/analytics/pageview/savePageView.tsindex ae702211c..6682edadc 100644--- a/queries/analytics/pageview/savePageView.ts+++ b/queries/analytics/pageview/savePageView.ts@@ -17,6 +17,9 @@ export async function savePageView(args: {screen?: string;language?: string;country?: string;+ subdivision1?: string;+ subdivision2?: string;+ city?: string;}) {return runQuery({[PRISMA]: () => relationalQuery(args),@@ -45,19 +48,32 @@ async function relationalQuery(data: {}async function clickhouseQuery(data) {- const { websiteId, id: sessionId, url, referrer, country, ...args } = data;- const website = await cache.fetchWebsite(websiteId);+ const {+ websiteId,+ id: sessionId,+ url,+ referrer,+ country,+ subdivision1,+ subdivision2,+ city,+ ...args+ } = data;const { getDateFormat, sendMessage } = kafka;+ const website = await cache.fetchWebsite(websiteId);const message = {- session_id: sessionId,website_id: websiteId,- url: url?.substring(0, URL_LENGTH),- referrer: referrer?.substring(0, URL_LENGTH),+ session_id: sessionId,rev_id: website?.revId || 0,- created_at: getDateFormat(new Date()),country: country ? country : null,+ subdivision1: subdivision1 ? subdivision1 : null,+ subdivision2: subdivision2 ? subdivision2 : null,+ city: city ? city : null,+ url: url?.substring(0, URL_LENGTH),+ referrer: referrer?.substring(0, URL_LENGTH),event_type: EVENT_TYPE.pageView,+ created_at: getDateFormat(new Date()),...args,};diff --git a/queries/analytics/session/createSession.ts b/queries/analytics/session/createSession.tsindex fe15f11c4..3afa9c9a4 100644--- a/queries/analytics/session/createSession.ts+++ b/queries/analytics/session/createSession.ts@@ -31,8 +31,24 @@ async function clickhouseQuery(data: {screen?: string;language?: string;country?: string;+ subdivision1?: string;+ subdivision2?: string;+ city?: string;}) {- const { id, websiteId, hostname, browser, os, device, screen, language, country } = data;+ const {+ id,+ websiteId,+ hostname,+ browser,+ os,+ device,+ screen,+ language,+ country,+ subdivision1,+ subdivision2,+ city,+ } = data;const { getDateFormat, sendMessage } = kafka;const website = await cache.fetchWebsite(websiteId);@@ -46,6 +62,9 @@ async function clickhouseQuery(data: {screen,language,country,+ subdivision1,+ subdivision2,+ city,rev_id: website?.revId || 0,created_at: getDateFormat(new Date()),};diff --git a/queries/analytics/session/getSession.ts b/queries/analytics/session/getSession.tsindex 09b238f21..a3400d265 100644--- a/queries/analytics/session/getSession.ts+++ b/queries/analytics/session/getSession.ts@@ -31,7 +31,10 @@ async function clickhouseQuery({ id: sessionId }: { id: string }) {device,screen,language,- country+ country,+ subdivision1,+ subdivision2,+ cityfrom eventwhere session_id = {sessionId:UUID}limit 1`,diff --git a/queries/analytics/session/getSessions.ts b/queries/analytics/session/getSessions.tsindex 460572778..910efc30b 100644--- a/queries/analytics/session/getSessions.ts+++ b/queries/analytics/session/getSessions.ts@@ -40,7 +40,10 @@ async function clickhouseQuery(websites: string[], startAt: Date) {device,screen,language,- country+ country,+ subdivision1,+ subdivision2,+ cityfrom eventwhere ${websites && websites.length > 0 ? `website_id in {websites:Array(UUID)}` : '0 = 0'}and created_at >= {startAt:DateTime('UTC')}`,diff --git a/scripts/build-geo.js b/scripts/build-geo.jsindex e84c15644..8a096f4b2 100644--- a/scripts/build-geo.js+++ b/scripts/build-geo.js@@ -12,7 +12,7 @@ let url =if (process.env.MAXMIND_LICENSE_KEY) {url =`https://download.maxmind.com/app/geoip_download` +- `?edition_id=GeoLite2-Country&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`;+ `?edition_id=GeoLite2-City&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`;}const dest = path.resolve(__dirname, '../node_modules/.geo');diff --git a/scripts/test.js b/scripts/test.jsindex a913f1b01..c1eb9b3b9 100644--- a/scripts/test.js+++ b/scripts/test.js@@ -1,10 +1,7 @@/* eslint-disable no-console */require('dotenv').config();-const fs = require('fs');const path = require('path');-const https = require('https');-const zlib = require('zlib');-const tar = require('tar');+const maxmind = require('maxmind');async function getLocation() {