Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
update kafka messaging
Brian Cao46b4b98d407c
diff --git a/lib/db.js b/lib/db.jsindex ff62da03d..30fa79f68 100644--- a/lib/db.js+++ b/lib/db.js@@ -2,7 +2,7 @@ import { PrismaClient } from '@prisma/client';import { ClickHouse } from 'clickhouse';import dateFormat from 'dateformat';import chalk from 'chalk';-import { getKafkaService } from './kafka';+import { getKafkaService } from './db/kafka';import {MYSQL,MYSQL_DATE_FORMATS,diff --git a/lib/kafka.js b/lib/db/kafka.jssimilarity index 69%rename from lib/kafka.jsrename to lib/db/kafka.jsindex 988062b9e..e2256d271 100644--- a/lib/kafka.js+++ b/lib/db/kafka.js@@ -1,4 +1,4 @@-import { Kafka } from 'kafkajs';+import { Kafka, logLevel } from 'kafkajs';import dateFormat from 'dateformat';export function getKafkaClient() {@@ -14,6 +14,7 @@ export function getKafkaClient() {clientId: 'umami',brokers: brokers,connectionTimeout: 3000,+ logLevel: logLevel.ERROR,});} else {return new Kafka({@@ -29,20 +30,29 @@ export function getKafkaClient() {});}}-const kafka = global.kafka || getKafkaClient();+let kafkaProducer = null;-if (process.env.NODE_ENV !== 'production') {- global.kafka = kafka;-}+(async () => {+ kafkaProducer = global.kakfaProducer || (await getKafkaProducer());++ if (process.env.NODE_ENV !== 'production') {+ global.kafka = kafka;+ global.kakfaProducer = kafkaProducer;+ }+})();-export { kafka };+export { kafka, kafkaProducer };-export async function kafkaProducer(params, topic) {+export async function getKafkaProducer() {const producer = kafka.producer();await producer.connect();- await producer.send({+ return producer;+}++export async function sendKafkaMessage(params, topic) {+ await kafkaProducer.send({topic,messages: [{@@ -50,6 +60,7 @@ export async function kafkaProducer(params, topic) {value: JSON.stringify(params),},],+ acks: 0,});}diff --git a/pages/api/heartbeat.js b/pages/api/heartbeat.jsnew file mode 100644index 000000000..a53213ada--- /dev/null+++ b/pages/api/heartbeat.js@@ -0,0 +1,5 @@+import { ok } from 'lib/response';++export default async (req, res) => {+ return ok(res, 'nice');+};diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.jsindex 0786042c7..ff4d52560 100644--- a/queries/analytics/event/saveEvent.js+++ b/queries/analytics/event/saveEvent.js@@ -6,7 +6,7 @@ import {runAnalyticsQuery,runQuery,} from 'lib/db';-import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';+import { sendKafkaMessage, getDateFormatKafka } from 'lib/db/kafka';export async function saveEvent(...args) {return runAnalyticsQuery({@@ -66,5 +66,5 @@ async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_namevent_name: event_name?.substr(0, 50),};- await kafkaProducer(params, 'event');+ await sendKafkaMessage(params, 'event');}diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.jsindex 0dd93b44e..af0d88f60 100644--- a/queries/analytics/pageview/savePageView.js+++ b/queries/analytics/pageview/savePageView.js@@ -6,7 +6,7 @@ import {runAnalyticsQuery,runQuery,} from 'lib/db';-import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';+import { sendKafkaMessage, getDateFormatKafka } from 'lib/db/kafka';export async function savePageView(...args) {return runAnalyticsQuery({@@ -54,5 +54,5 @@ async function kafkaQuery(website_id, { session_uuid, url, referrer }) {referrer: referrer?.substr(0, URL_LENGTH),};- await kafkaProducer(params, 'pageview');+ await sendKafkaMessage(params, 'pageview');}diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.jsindex 5c4b982c0..565d45a28 100644--- a/queries/analytics/session/createSession.js+++ b/queries/analytics/session/createSession.js@@ -6,7 +6,7 @@ import {runAnalyticsQuery,runQuery,} from 'lib/db';-import { kafkaProducer, getDateFormatKafka } from 'lib/kafka';+import { sendKafkaMessage, getDateFormatKafka } from 'lib/db/kafka';import { getSessionByUuid } from 'queries';export async function createSession(...args) {@@ -73,7 +73,7 @@ async function kafkaQuery(country: country ? country : null,};- await kafkaProducer(params, 'session');+ await sendKafkaMessage(params, 'session');return getSessionByUuid(session_uuid);}