📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Replace blake3 hash with 128bit xxHash
Blue Moused97f0f23c015
diff --git a/backend/apps/production/src/analytics/analytics.controller.ts b/backend/apps/production/src/analytics/analytics.controller.tsindex e52df0439..f234f7d1e 100644--- a/backend/apps/production/src/analytics/analytics.controller.ts+++ b/backend/apps/production/src/analytics/analytics.controller.ts@@ -9,7 +9,6 @@ import _uniqBy from 'lodash/uniqBy'import dayjs from 'dayjs'import utc from 'dayjs/plugin/utc'import dayjsTimezone from 'dayjs/plugin/timezone'-import { hash } from 'blake3'import {ApiOperation,ApiOkResponse,@@ -71,6 +70,7 @@ import {checkRateLimit,getGeoDetails,getIPFromHeaders,+ hash,} from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'@@ -113,7 +113,7 @@ const getSessionKeyCustom = (pid: string,ev: string,salt = '',-) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`+) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`)}`// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.tsindex a776f714f..c7e08be97 100644--- a/backend/apps/production/src/analytics/analytics.service.ts+++ b/backend/apps/production/src/analytics/analytics.service.ts@@ -28,7 +28,6 @@ import dayjsTimezone from 'dayjs/plugin/timezone'import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'import ipRangeCheck from 'ip-range-check'import validator from 'validator'-import { hash } from 'blake3'import {Injectable,BadRequestException,@@ -62,6 +61,7 @@ import {import { clickhouse } from '../common/integrations/clickhouse'import {calculateRelativePercentage,+ hash,millisecondsToSeconds,sumArrays,} from '../common/utils'@@ -110,7 +110,7 @@ dayjs.extend(dayjsTimezone)dayjs.extend(isSameOrBefore)export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>- `ses_${hash(`${ua}${ip}${pid}${salt}`).toString('hex')}`+ `ses_${hash(`${ua}${ip}${pid}${salt}`)}`export const getHeartbeatKey = (pid: string, sessionID: string) =>`hb:${pid}:${sessionID}`@@ -3983,9 +3983,10 @@ export class AnalyticsService {getErrorID(errorDTO: ErrorDTO): string {const { name, message, colno, lineno, filename } = errorDTO- return hash(`${name}${message}${colno}${lineno}${filename}`, {- length: 16,- }).toString('hex')+ return hash(`${name}${message}${colno}${lineno}${filename}`).substring(+ 0,+ 16,+ )}async validateEIDs(eids: string[], pid: string) {diff --git a/backend/apps/production/src/captcha/captcha.service.ts b/backend/apps/production/src/captcha/captcha.service.tsindex d0b7f6a6f..b3a5af882 100644--- a/backend/apps/production/src/captcha/captcha.service.ts+++ b/backend/apps/production/src/captcha/captcha.service.ts@@ -1,5 +1,4 @@import { Injectable, BadRequestException } from '@nestjs/common'-import { hash } from 'blake3'import svgCaptcha from 'svg-captcha'import CryptoJS from 'crypto-js'import _toLower from 'lodash/toLower'@@ -18,7 +17,7 @@ import {CAPTCHA_SALT,CAPTCHA_TOKEN_LIFETIME,} from '../common/constants'-import { getGeoDetails } from '../common/utils'+import { getGeoDetails, hash } from '../common/utils'import { GeneratedCaptcha } from './interfaces/generated-captcha'import { captchaTransformer } from './utils/transformers'import { clickhouse } from '../common/integrations/clickhouse'@@ -205,7 +204,7 @@ export class CaptchaService {}hashCaptcha(text: string): string {- return hash(captchaString(text)).toString('hex')+ return hash(captchaString(text))}async generateCaptcha(theme: string): Promise<GeneratedCaptcha> {diff --git a/backend/apps/production/src/common/constants.ts b/backend/apps/production/src/common/constants.tsindex 506f48316..d16c5a881 100644--- a/backend/apps/production/src/common/constants.ts+++ b/backend/apps/production/src/common/constants.ts@@ -1,8 +1,8 @@import Redis from 'ioredis'import path from 'path'-import { hash } from 'blake3'import _toNumber from 'lodash/toNumber'import 'dotenv/config'+import { hash } from './utils'const { PAYPAL_CLIENT_ID, PAYPAL_CLIENT_SECRET, EMAIL_ACTION_ENCRYPTION_KEY } =process.envdiff --git a/backend/apps/production/src/common/utils.ts b/backend/apps/production/src/common/utils.tsindex 2f6bcb6b2..dabcc823d 100644--- a/backend/apps/production/src/common/utils.ts+++ b/backend/apps/production/src/common/utils.ts@@ -1,8 +1,8 @@+import { xxh3 } from '@node-rs/xxhash'import path from 'path'import fs from 'fs'import { Reader, CityResponse } from 'maxmind'import { HttpException } from '@nestjs/common'-import { hash } from 'blake3'import timezones from 'countries-and-timezones'import randomstring from 'randomstring'import _sample from 'lodash/sample'@@ -15,6 +15,13 @@ import _split from 'lodash/split'import { redis, isDevelopment, isProxiedByCloudflare } from './constants'+/*+ Returns 128 bit (a string of 32 chars) hash of the provided string using xxHash algo.+*/+export const hash = (content: string): string => {+ return xxh3.xxh128(content).toString(16)+}+const marketingTips = {en: ['Blog frequently and consistently to build search engine presence.\nTo get website visitors, you need lots and lots of content. Your prospects are using Google every day to search for what they want. One of the most reliable ways to show up in Google results is to blog about lots of topics your customers care about.',@@ -41,9 +48,9 @@ const RATE_LIMIT_REQUESTS_AMOUNT = 3const RATE_LIMIT_TIMEOUT = 86400 // 24 hoursconst getRateLimitHash = (ipOrApiKey: string, salt = '') =>- `rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`+ `rl:${hash(`${ipOrApiKey}${salt}`)}`-const getRandomTip = (language = 'en'): string => {+export const getRandomTip = (language = 'en'): string => {return _sample(marketingTips[language])}@@ -56,7 +63,7 @@ const formatterLookup = [{ value: 1e9, symbol: 'B' },]-const nFormatter = (num: any, digits = 1) => {+export const nFormatter = (num: any, digits = 1) => {const item = _find(formatterLookup.slice().reverse(),({ value }) => Math.abs(num) >= value,@@ -68,7 +75,7 @@ const nFormatter = (num: any, digits = 1) => {}// 'action' is used as a salt to differ rate limiting routes-const checkRateLimit = async (+export const checkRateLimit = async (ip: string,action: string,reqAmount: number = RATE_LIMIT_REQUESTS_AMOUNT,@@ -108,7 +115,7 @@ export const checkRateLimitForApiKey = async (* @param newVal The value that changed* @param round Numbers after floating point*/-const calculateRelativePercentage = (+export const calculateRelativePercentage = (oldVal: number,newVal: number,round = 2,@@ -124,23 +131,24 @@ const calculateRelativePercentage = (return _round((1 - newVal / oldVal) * -100, round)}-const generateRecoveryCode = () =>+export const generateRecoveryCode = () =>randomstring.generate({length: 30,charset: 'alphabetic',capitalization: 'uppercase',})-const generateRefCode = () =>+export const generateRefCode = () =>randomstring.generate({length: 8,charset: 'alphanumeric',capitalization: 'uppercase',})-const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000+export const millisecondsToSeconds = (milliseconds: number) =>+ milliseconds / 1000-const generateRandomString = (length: number): string =>+export const generateRandomString = (length: number): string =>randomstring.generate(length)const dummyLookup = () => ({@@ -190,7 +198,7 @@ interface IPGeoDetails {city: string | null}-const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {+export const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {// Stage 1: Using IP address based geo lookupconst data = lookup.get(ip)@@ -217,7 +225,10 @@ const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {}}-const getIPFromHeaders = (headers: any, tryXClientIPAddress?: boolean) => {+export const getIPFromHeaders = (+ headers: any,+ tryXClientIPAddress?: boolean,+) => {if (tryXClientIPAddress && headers['x-client-ip-address']) {return headers['x-client-ip-address']}@@ -243,7 +254,7 @@ const getIPFromHeaders = (headers: any, tryXClientIPAddress?: boolean) => {return _split(ip, ',')[0]}-const sumArrays = (source: number[], target: number[]) => {+export const sumArrays = (source: number[], target: number[]) => {const result = []const size = _size(source)@@ -254,17 +265,4 @@ const sumArrays = (source: number[], target: number[]) => {return result}-export {- getRandomTip,- checkRateLimit,- generateRecoveryCode,- calculateRelativePercentage,- millisecondsToSeconds,- generateRandomString,- nFormatter,- lookup,- getGeoDetails,- getIPFromHeaders,- generateRefCode,- sumArrays,-}+export { lookup }diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.tsindex 8943177c5..8aa9acbe7 100644--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts@@ -8,7 +8,6 @@ import _size from 'lodash/size'import dayjs from 'dayjs'import utc from 'dayjs/plugin/utc'import dayjsTimezone from 'dayjs/plugin/timezone'-import { hash } from 'blake3'import {Controller,Body,@@ -61,6 +60,7 @@ import {checkRateLimit,getGeoDetails,getIPFromHeaders,+ hash,} from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'@@ -97,7 +97,7 @@ const getSessionKeyCustom = (pid: string,ev: string,salt = '',-) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`+) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`)}`// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.tsindex abf54459b..023c6a4e3 100644--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts@@ -28,7 +28,6 @@ import dayjsTimezone from 'dayjs/plugin/timezone'import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'import ipRangeCheck from 'ip-range-check'import validator from 'validator'-import { hash } from 'blake3'import {Injectable,BadRequestException,@@ -54,6 +53,7 @@ import {import { clickhouse } from '../common/integrations/clickhouse'import {getFunnelsClickhouse,+ hash,millisecondsToSeconds,sumArrays,} from '../common/utils'@@ -99,7 +99,7 @@ dayjs.extend(dayjsTimezone)dayjs.extend(isSameOrBefore)export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>- `ses_${hash(`${ua}${ip}${pid}${salt}`).toString('hex')}`+ `ses_${hash(`${ua}${ip}${pid}${salt}`)}`export const getHeartbeatKey = (pid: string, sessionID: string) =>`hb:${pid}:${sessionID}`@@ -3370,9 +3370,10 @@ export class AnalyticsService {getErrorID(errorDTO: ErrorDTO): string {const { name, message, colno, lineno, filename } = errorDTO- return hash(`${name}${message}${colno}${lineno}${filename}`, {- length: 16,- }).toString('hex')+ return hash(`${name}${message}${colno}${lineno}${filename}`).substring(+ 0,+ 16,+ )}async validateEIDs(eids: string[], pid: string) {diff --git a/backend/apps/selfhosted/src/common/utils.ts b/backend/apps/selfhosted/src/common/utils.tsindex 03da4268a..65038c845 100644--- a/backend/apps/selfhosted/src/common/utils.ts+++ b/backend/apps/selfhosted/src/common/utils.ts@@ -1,6 +1,6 @@import { NotFoundException, HttpException } from '@nestjs/common'+import { xxh3 } from '@node-rs/xxhash'import timezones from 'countries-and-timezones'-import { hash } from 'blake3'import { v5 as uuidv5 } from 'uuid'import _join from 'lodash/join'import _filter from 'lodash/filter'@@ -34,6 +34,10 @@ import { ProjectViewEntity } from '../project/entity/project-view.entity'dayjs.extend(utc)+export const hash = (content: string): string => {+ return xxh3.xxh128(content).toString(16)+}+const RATE_LIMIT_REQUESTS_AMOUNT = 3const RATE_LIMIT_TIMEOUT = 86400 // 24 hours@@ -50,7 +54,7 @@ const ALLOWED_KEYS = [const ALLOWED_FUNNEL_KEYS = ['name', 'steps']const getRateLimitHash = (ipOrApiKey: string, salt = '') =>- `rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`+ `rl:${hash(`${ipOrApiKey}${salt}`)}`// 'action' is used as a salt to differ rate limiting routesconst checkRateLimit = async (diff --git a/backend/package.json b/backend/package.jsonindex 08b5eeb20..7a0ba5a18 100644--- a/backend/package.json+++ b/backend/package.json@@ -45,6 +45,7 @@"@nestjs/swagger": "^7.4.2","@nestjs/terminus": "^10.2.3","@nestjs/typeorm": "^10.0.2",+ "@node-rs/xxhash": "^1.7.4","@paypal/payouts-sdk": "^1.1.1","@sentry/cli": "^2.37.0","@sentry/nestjs": "^8.34.0",