Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Use Clickhouse for salt repo for Swetrix CE

Blue Mouse4743ad61de41
diff --git a/backend/apps/community/src/analytics/analytics.module.ts b/backend/apps/community/src/analytics/analytics.module.ts
index f9de1a0ff..8b78bd0e7 100644
--- a/backend/apps/community/src/analytics/analytics.module.ts
+++ b/backend/apps/community/src/analytics/analytics.module.ts
@@ -1,16 +1,14 @@
import { Module } from '@nestjs/common'
-import { TypeOrmModule } from '@nestjs/typeorm'
import { AnalyticsService } from './analytics.service'
import { AnalyticsController } from './analytics.controller'
import { HeartbeatGateway } from './heartbeat.gateway'
import { SaltService } from './salt.service'
-import { Salt } from './entities/salt.entity'
import { AppLoggerModule } from '../logger/logger.module'
import { ProjectModule } from '../project/project.module'
@Module({
- imports: [TypeOrmModule.forFeature([Salt]), AppLoggerModule, ProjectModule],
+ imports: [AppLoggerModule, ProjectModule],
providers: [AnalyticsService, SaltService, HeartbeatGateway],
exports: [AnalyticsService, SaltService],
controllers: [AnalyticsController],
diff --git a/backend/apps/community/src/analytics/entities/salt.entity.ts b/backend/apps/community/src/analytics/entities/salt.entity.ts
deleted file mode 100644
index 4ba00304c..000000000
--- a/backend/apps/community/src/analytics/entities/salt.entity.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import { Column, Entity, PrimaryColumn } from 'typeorm'
-
-import { SaltRotationType } from '../salt.service'
-
-@Entity()
-export class Salt {
- @PrimaryColumn('varchar', { length: 10 })
- rotation: SaltRotationType
-
- @Column('text')
- salt: string
-
- @Column({ type: 'timestamp' })
- expiresAt: Date
-
- @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
- created: Date
-}
diff --git a/backend/apps/community/src/analytics/salt.service.ts b/backend/apps/community/src/analytics/salt.service.ts
index ea72486fb..393cd5db5 100644
--- a/backend/apps/community/src/analytics/salt.service.ts
+++ b/backend/apps/community/src/analytics/salt.service.ts
@@ -1,12 +1,10 @@
import { Injectable, Logger } from '@nestjs/common'
-import { InjectRepository } from '@nestjs/typeorm'
-import { Repository } from 'typeorm'
import * as bcrypt from 'bcrypt'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import { redis } from '../common/constants'
-import { Salt } from './entities/salt.entity'
+import { getSaltClickhouse, saveSaltClickhouse } from '../common/utils'
dayjs.extend(utc)
@@ -36,10 +34,7 @@ const getRedisKeyForRotation = (rotation: SaltRotationType): string => {
export class SaltService {
private readonly logger = new Logger(SaltService.name)
- constructor(
- @InjectRepository(Salt)
- private saltRepository: Repository<Salt>,
- ) {}
+ constructor() {}
private async generateSalt(): Promise<string> {
return bcrypt.genSalt(10)
@@ -67,13 +62,11 @@ export class SaltService {
return cachedSalt
}
- // Check MySQL
- const now = dayjs.utc().toDate()
- const existingSalt = await this.saltRepository.findOne({
- where: { rotation },
- })
+ // Check ClickHouse
+ const now = dayjs.utc()
+ const existingSalt = await getSaltClickhouse(rotation)
- if (existingSalt && existingSalt.expiresAt > now) {
+ if (existingSalt && dayjs.utc(existingSalt.expiresAt).isAfter(now)) {
// Cache in Redis with 10-minute TTL
await redis.set(redisKey, existingSalt.salt, 'EX', REDIS_CACHE_TTL)
return existingSalt.salt
@@ -81,13 +74,14 @@ export class SaltService {
// Generate new salt
const salt = await this.generateSalt()
- const expiresAt = getEndOfPeriod(rotation).toDate()
+ const expiresAt = getEndOfPeriod(rotation)
- // Save to MySQL (upsert)
- await this.saltRepository.save({
+ // Save to ClickHouse (upsert)
+ await saveSaltClickhouse({
rotation,
salt,
- expiresAt,
+ expiresAt: expiresAt.format('YYYY-MM-DD HH:mm:ss'),
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
})
// Cache in Redis with 10-minute TTL
@@ -102,21 +96,20 @@ export class SaltService {
async regenerateExpiredSalts(): Promise<void> {
const rotations: SaltRotationType[] = ['daily', 'monthly']
- const now = dayjs.utc().toDate()
+ const now = dayjs.utc()
for (const rotation of rotations) {
- const existingSalt = await this.saltRepository.findOne({
- where: { rotation },
- })
+ const existingSalt = await getSaltClickhouse(rotation)
- if (!existingSalt || existingSalt.expiresAt <= now) {
+ if (!existingSalt || dayjs.utc(existingSalt.expiresAt).isBefore(now)) {
const salt = await this.generateSalt()
- const expiresAt = getEndOfPeriod(rotation).toDate()
+ const expiresAt = getEndOfPeriod(rotation)
- await this.saltRepository.save({
+ await saveSaltClickhouse({
rotation,
salt,
- expiresAt,
+ expiresAt: expiresAt.format('YYYY-MM-DD HH:mm:ss'),
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
})
// Cache in Redis with 10-minute TTL
diff --git a/backend/apps/community/src/common/types.ts b/backend/apps/community/src/common/types.ts
index fc13566c0..980e28ae8 100644
--- a/backend/apps/community/src/common/types.ts
+++ b/backend/apps/community/src/common/types.ts
@@ -39,3 +39,10 @@ export interface ClickhouseAnnotation {
projectId: string
created: string
}
+
+export interface ClickhouseSalt {
+ rotation: string
+ salt: string
+ expiresAt: string
+ created: string
+}
diff --git a/backend/apps/community/src/common/utils.ts b/backend/apps/community/src/common/utils.ts
index 6b59ec2ec..cd5781e94 100644
--- a/backend/apps/community/src/common/utils.ts
+++ b/backend/apps/community/src/common/utils.ts
@@ -30,7 +30,7 @@ import { clickhouse } from './integrations/clickhouse'
import { BotsProtectionLevel, Project } from '../project/entity/project.entity'
import { ProjectViewEntity } from '../project/entity/project-view.entity'
-import { ClickhouseFunnel, ClickhouseAnnotation } from './types'
+import { ClickhouseFunnel, ClickhouseAnnotation, ClickhouseSalt } from './types'
dayjs.extend(utc)
@@ -1224,6 +1224,56 @@ const unpinProjectClickhouse = async (
})
}
+// Salt functions
+const getSaltClickhouse = async (
+ rotation: string,
+): Promise<ClickhouseSalt | null> => {
+ const query = `SELECT * FROM salt WHERE rotation = {rotation:String}`
+
+ try {
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: { rotation },
+ })
+ .then(resultSet => resultSet.json<ClickhouseSalt>())
+
+ return _head(data) || null
+ } catch {
+ return null
+ }
+}
+
+const saveSaltClickhouse = async (saltData: ClickhouseSalt) => {
+ const existing = await getSaltClickhouse(saltData.rotation)
+
+ if (existing) {
+ const query = `ALTER TABLE salt UPDATE salt = {salt:String}, expiresAt = {expiresAt:String} WHERE rotation = {rotation:String}`
+ await clickhouse.command({
+ query,
+ query_params: {
+ salt: saltData.salt,
+ expiresAt: saltData.expiresAt,
+ rotation: saltData.rotation,
+ },
+ })
+ } else {
+ await clickhouse.insert({
+ table: 'salt',
+ format: 'JSONEachRow',
+ values: [saltData],
+ })
+ }
+}
+
+const deleteSaltClickhouse = async (rotation: string) => {
+ const query = `ALTER TABLE salt DELETE WHERE rotation = {rotation:String}`
+ await clickhouse.command({
+ query,
+ query_params: { rotation },
+ })
+}
+
export {
checkRateLimit,
createProjectClickhouse,
@@ -1272,4 +1322,8 @@ export {
getPinnedProjectsClickhouse,
pinProjectClickhouse,
unpinProjectClickhouse,
+ // salt
+ getSaltClickhouse,
+ saveSaltClickhouse,
+ deleteSaltClickhouse,
}
diff --git a/backend/apps/community/src/task-manager/task-manager.service.ts b/backend/apps/community/src/task-manager/task-manager.service.ts
index e527db22f..c87add44b 100644
--- a/backend/apps/community/src/task-manager/task-manager.service.ts
+++ b/backend/apps/community/src/task-manager/task-manager.service.ts
@@ -1,15 +1,11 @@
import { Injectable } from '@nestjs/common'
import { Cron, CronExpression } from '@nestjs/schedule'
-import { AppLoggerService } from '../logger/logger.service'
import { SaltService } from '../analytics/salt.service'
@Injectable()
export class TaskManagerService {
- constructor(
- private readonly logger: AppLoggerService,
- private readonly saltService: SaltService,
- ) {}
+ constructor(private readonly saltService: SaltService) {}
@Cron(CronExpression.EVERY_HOUR)
async regenerateGlobalSalts() {
diff --git a/backend/migrations/clickhouse/initialise_selfhosted.js b/backend/migrations/clickhouse/initialise_selfhosted.js
index 447ba8964..faa094c54 100644
--- a/backend/migrations/clickhouse/initialise_selfhosted.js
+++ b/backend/migrations/clickhouse/initialise_selfhosted.js
@@ -106,6 +106,16 @@ const CLICKHOUSE_INIT_QUERIES = [
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
+
+ `CREATE TABLE IF NOT EXISTS ${dbName}.salt
+ (
+ rotation String CODEC(ZSTD(3)),
+ salt String CODEC(ZSTD(3)),
+ expiresAt DateTime CODEC(Delta(4), LZ4),
+ created DateTime CODEC(Delta(4), LZ4)
+ )
+ ENGINE = MergeTree()
+ PRIMARY KEY rotation;`,
]
const initialiseSelfhosted = async () => {
diff --git a/backend/migrations/clickhouse/selfhosted_2026_01_02_salt.js b/backend/migrations/clickhouse/selfhosted_2026_01_02_salt.js
new file mode 100644
index 000000000..b6993898f
--- /dev/null
+++ b/backend/migrations/clickhouse/selfhosted_2026_01_02_salt.js
@@ -0,0 +1,16 @@
+// This migration creates the salt table
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ `CREATE TABLE IF NOT EXISTS ${dbName}.salt
+ (
+ rotation String CODEC(ZSTD(3)),
+ salt String CODEC(ZSTD(3)),
+ expiresAt DateTime CODEC(Delta(4), LZ4),
+ created DateTime CODEC(Delta(4), LZ4)
+ )
+ ENGINE = MergeTree()
+ PRIMARY KEY rotation;`,
+]
+
+queriesRunner(queries)