Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Replace blake3 hash with 128bit xxHash

Blue Moused97f0f23c015
diff --git a/backend/apps/production/src/analytics/analytics.controller.ts b/backend/apps/production/src/analytics/analytics.controller.ts
index e52df0439..f234f7d1e 100644
--- a/backend/apps/production/src/analytics/analytics.controller.ts
+++ b/backend/apps/production/src/analytics/analytics.controller.ts
@@ -9,7 +9,6 @@ import _uniqBy from 'lodash/uniqBy'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import dayjsTimezone from 'dayjs/plugin/timezone'
-import { hash } from 'blake3'
import {
ApiOperation,
ApiOkResponse,
@@ -71,6 +70,7 @@ import {
checkRateLimit,
getGeoDetails,
getIPFromHeaders,
+ hash,
} from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
@@ -113,7 +113,7 @@ const getSessionKeyCustom = (
pid: string,
ev: string,
salt = '',
-) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`
+) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`)}`
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.ts
index a776f714f..c7e08be97 100644
--- a/backend/apps/production/src/analytics/analytics.service.ts
+++ b/backend/apps/production/src/analytics/analytics.service.ts
@@ -28,7 +28,6 @@ import dayjsTimezone from 'dayjs/plugin/timezone'
import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'
import ipRangeCheck from 'ip-range-check'
import validator from 'validator'
-import { hash } from 'blake3'
import {
Injectable,
BadRequestException,
@@ -62,6 +61,7 @@ import {
import { clickhouse } from '../common/integrations/clickhouse'
import {
calculateRelativePercentage,
+ hash,
millisecondsToSeconds,
sumArrays,
} from '../common/utils'
@@ -110,7 +110,7 @@ dayjs.extend(dayjsTimezone)
dayjs.extend(isSameOrBefore)
export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>
- `ses_${hash(`${ua}${ip}${pid}${salt}`).toString('hex')}`
+ `ses_${hash(`${ua}${ip}${pid}${salt}`)}`
export const getHeartbeatKey = (pid: string, sessionID: string) =>
`hb:${pid}:${sessionID}`
@@ -3983,9 +3983,10 @@ export class AnalyticsService {
getErrorID(errorDTO: ErrorDTO): string {
const { name, message, colno, lineno, filename } = errorDTO
- return hash(`${name}${message}${colno}${lineno}${filename}`, {
- length: 16,
- }).toString('hex')
+ return hash(`${name}${message}${colno}${lineno}${filename}`).substring(
+ 0,
+ 16,
+ )
}
async validateEIDs(eids: string[], pid: string) {
diff --git a/backend/apps/production/src/captcha/captcha.service.ts b/backend/apps/production/src/captcha/captcha.service.ts
index d0b7f6a6f..b3a5af882 100644
--- a/backend/apps/production/src/captcha/captcha.service.ts
+++ b/backend/apps/production/src/captcha/captcha.service.ts
@@ -1,5 +1,4 @@
import { Injectable, BadRequestException } from '@nestjs/common'
-import { hash } from 'blake3'
import svgCaptcha from 'svg-captcha'
import CryptoJS from 'crypto-js'
import _toLower from 'lodash/toLower'
@@ -18,7 +17,7 @@ import {
CAPTCHA_SALT,
CAPTCHA_TOKEN_LIFETIME,
} from '../common/constants'
-import { getGeoDetails } from '../common/utils'
+import { getGeoDetails, hash } from '../common/utils'
import { GeneratedCaptcha } from './interfaces/generated-captcha'
import { captchaTransformer } from './utils/transformers'
import { clickhouse } from '../common/integrations/clickhouse'
@@ -205,7 +204,7 @@ export class CaptchaService {
}
hashCaptcha(text: string): string {
- return hash(captchaString(text)).toString('hex')
+ return hash(captchaString(text))
}
async generateCaptcha(theme: string): Promise<GeneratedCaptcha> {
diff --git a/backend/apps/production/src/common/constants.ts b/backend/apps/production/src/common/constants.ts
index 506f48316..d16c5a881 100644
--- a/backend/apps/production/src/common/constants.ts
+++ b/backend/apps/production/src/common/constants.ts
@@ -1,8 +1,8 @@
import Redis from 'ioredis'
import path from 'path'
-import { hash } from 'blake3'
import _toNumber from 'lodash/toNumber'
import 'dotenv/config'
+import { hash } from './utils'
const { PAYPAL_CLIENT_ID, PAYPAL_CLIENT_SECRET, EMAIL_ACTION_ENCRYPTION_KEY } =
process.env
diff --git a/backend/apps/production/src/common/utils.ts b/backend/apps/production/src/common/utils.ts
index 2f6bcb6b2..dabcc823d 100644
--- a/backend/apps/production/src/common/utils.ts
+++ b/backend/apps/production/src/common/utils.ts
@@ -1,8 +1,8 @@
+import { xxh3 } from '@node-rs/xxhash'
import path from 'path'
import fs from 'fs'
import { Reader, CityResponse } from 'maxmind'
import { HttpException } from '@nestjs/common'
-import { hash } from 'blake3'
import timezones from 'countries-and-timezones'
import randomstring from 'randomstring'
import _sample from 'lodash/sample'
@@ -15,6 +15,13 @@ import _split from 'lodash/split'
import { redis, isDevelopment, isProxiedByCloudflare } from './constants'
+/*
+ Returns 128 bit (a string of 32 chars) hash of the provided string using xxHash algo.
+*/
+export const hash = (content: string): string => {
+ return xxh3.xxh128(content).toString(16)
+}
+
const marketingTips = {
en: [
'Blog frequently and consistently to build search engine presence.\nTo get website visitors, you need lots and lots of content. Your prospects are using Google every day to search for what they want. One of the most reliable ways to show up in Google results is to blog about lots of topics your customers care about.',
@@ -41,9 +48,9 @@ const RATE_LIMIT_REQUESTS_AMOUNT = 3
const RATE_LIMIT_TIMEOUT = 86400 // 24 hours
const getRateLimitHash = (ipOrApiKey: string, salt = '') =>
- `rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`
+ `rl:${hash(`${ipOrApiKey}${salt}`)}`
-const getRandomTip = (language = 'en'): string => {
+export const getRandomTip = (language = 'en'): string => {
return _sample(marketingTips[language])
}
@@ -56,7 +63,7 @@ const formatterLookup = [
{ value: 1e9, symbol: 'B' },
]
-const nFormatter = (num: any, digits = 1) => {
+export const nFormatter = (num: any, digits = 1) => {
const item = _find(
formatterLookup.slice().reverse(),
({ value }) => Math.abs(num) >= value,
@@ -68,7 +75,7 @@ const nFormatter = (num: any, digits = 1) => {
}
// 'action' is used as a salt to differ rate limiting routes
-const checkRateLimit = async (
+export const checkRateLimit = async (
ip: string,
action: string,
reqAmount: number = RATE_LIMIT_REQUESTS_AMOUNT,
@@ -108,7 +115,7 @@ export const checkRateLimitForApiKey = async (
* @param newVal The value that changed
* @param round Numbers after floating point
*/
-const calculateRelativePercentage = (
+export const calculateRelativePercentage = (
oldVal: number,
newVal: number,
round = 2,
@@ -124,23 +131,24 @@ const calculateRelativePercentage = (
return _round((1 - newVal / oldVal) * -100, round)
}
-const generateRecoveryCode = () =>
+export const generateRecoveryCode = () =>
randomstring.generate({
length: 30,
charset: 'alphabetic',
capitalization: 'uppercase',
})
-const generateRefCode = () =>
+export const generateRefCode = () =>
randomstring.generate({
length: 8,
charset: 'alphanumeric',
capitalization: 'uppercase',
})
-const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000
+export const millisecondsToSeconds = (milliseconds: number) =>
+ milliseconds / 1000
-const generateRandomString = (length: number): string =>
+export const generateRandomString = (length: number): string =>
randomstring.generate(length)
const dummyLookup = () => ({
@@ -190,7 +198,7 @@ interface IPGeoDetails {
city: string | null
}
-const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
+export const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
// Stage 1: Using IP address based geo lookup
const data = lookup.get(ip)
@@ -217,7 +225,10 @@ const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
}
}
-const getIPFromHeaders = (headers: any, tryXClientIPAddress?: boolean) => {
+export const getIPFromHeaders = (
+ headers: any,
+ tryXClientIPAddress?: boolean,
+) => {
if (tryXClientIPAddress && headers['x-client-ip-address']) {
return headers['x-client-ip-address']
}
@@ -243,7 +254,7 @@ const getIPFromHeaders = (headers: any, tryXClientIPAddress?: boolean) => {
return _split(ip, ',')[0]
}
-const sumArrays = (source: number[], target: number[]) => {
+export const sumArrays = (source: number[], target: number[]) => {
const result = []
const size = _size(source)
@@ -254,17 +265,4 @@ const sumArrays = (source: number[], target: number[]) => {
return result
}
-export {
- getRandomTip,
- checkRateLimit,
- generateRecoveryCode,
- calculateRelativePercentage,
- millisecondsToSeconds,
- generateRandomString,
- nFormatter,
- lookup,
- getGeoDetails,
- getIPFromHeaders,
- generateRefCode,
- sumArrays,
-}
+export { lookup }
diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
index 8943177c5..8aa9acbe7 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -8,7 +8,6 @@ import _size from 'lodash/size'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import dayjsTimezone from 'dayjs/plugin/timezone'
-import { hash } from 'blake3'
import {
Controller,
Body,
@@ -61,6 +60,7 @@ import {
checkRateLimit,
getGeoDetails,
getIPFromHeaders,
+ hash,
} from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
@@ -97,7 +97,7 @@ const getSessionKeyCustom = (
pid: string,
ev: string,
salt = '',
-) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`
+) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`)}`
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.ts
index abf54459b..023c6a4e3 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts
@@ -28,7 +28,6 @@ import dayjsTimezone from 'dayjs/plugin/timezone'
import isSameOrBefore from 'dayjs/plugin/isSameOrBefore'
import ipRangeCheck from 'ip-range-check'
import validator from 'validator'
-import { hash } from 'blake3'
import {
Injectable,
BadRequestException,
@@ -54,6 +53,7 @@ import {
import { clickhouse } from '../common/integrations/clickhouse'
import {
getFunnelsClickhouse,
+ hash,
millisecondsToSeconds,
sumArrays,
} from '../common/utils'
@@ -99,7 +99,7 @@ dayjs.extend(dayjsTimezone)
dayjs.extend(isSameOrBefore)
export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>
- `ses_${hash(`${ua}${ip}${pid}${salt}`).toString('hex')}`
+ `ses_${hash(`${ua}${ip}${pid}${salt}`)}`
export const getHeartbeatKey = (pid: string, sessionID: string) =>
`hb:${pid}:${sessionID}`
@@ -3370,9 +3370,10 @@ export class AnalyticsService {
getErrorID(errorDTO: ErrorDTO): string {
const { name, message, colno, lineno, filename } = errorDTO
- return hash(`${name}${message}${colno}${lineno}${filename}`, {
- length: 16,
- }).toString('hex')
+ return hash(`${name}${message}${colno}${lineno}${filename}`).substring(
+ 0,
+ 16,
+ )
}
async validateEIDs(eids: string[], pid: string) {
diff --git a/backend/apps/selfhosted/src/common/utils.ts b/backend/apps/selfhosted/src/common/utils.ts
index 03da4268a..65038c845 100644
--- a/backend/apps/selfhosted/src/common/utils.ts
+++ b/backend/apps/selfhosted/src/common/utils.ts
@@ -1,6 +1,6 @@
import { NotFoundException, HttpException } from '@nestjs/common'
+import { xxh3 } from '@node-rs/xxhash'
import timezones from 'countries-and-timezones'
-import { hash } from 'blake3'
import { v5 as uuidv5 } from 'uuid'
import _join from 'lodash/join'
import _filter from 'lodash/filter'
@@ -34,6 +34,10 @@ import { ProjectViewEntity } from '../project/entity/project-view.entity'
dayjs.extend(utc)
+export const hash = (content: string): string => {
+ return xxh3.xxh128(content).toString(16)
+}
+
const RATE_LIMIT_REQUESTS_AMOUNT = 3
const RATE_LIMIT_TIMEOUT = 86400 // 24 hours
@@ -50,7 +54,7 @@ const ALLOWED_KEYS = [
const ALLOWED_FUNNEL_KEYS = ['name', 'steps']
const getRateLimitHash = (ipOrApiKey: string, salt = '') =>
- `rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`
+ `rl:${hash(`${ipOrApiKey}${salt}`)}`
// 'action' is used as a salt to differ rate limiting routes
const checkRateLimit = async (
diff --git a/backend/package.json b/backend/package.json
index 08b5eeb20..7a0ba5a18 100644
--- a/backend/package.json
+++ b/backend/package.json
@@ -45,6 +45,7 @@
"@nestjs/swagger": "^7.4.2",
"@nestjs/terminus": "^10.2.3",
"@nestjs/typeorm": "^10.0.2",
+ "@node-rs/xxhash": "^1.7.4",
"@paypal/payouts-sdk": "^1.1.1",
"@sentry/cli": "^2.37.0",
"@sentry/nestjs": "^8.34.0",