📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
some improvements
Blue Mousefd1849c23478
diff --git a/migrations/clickhouse/2023_03_11.js b/migrations/clickhouse/2023_03_11.jsnew file mode 100644index 000000000..a0999c7dd--- /dev/null+++ b/migrations/clickhouse/2023_03_11.js@@ -0,0 +1,19 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ `CREATE TABLE IF NOT EXISTS ${dbName}.captcha+ (+ pid FixedString(12),+ dv Nullable(String),+ br Nullable(String),+ os Nullable(String),+ cc Nullable(FixedString(2)),+ manuallyPassed UInt8,+ created DateTime+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,+]++queriesRunner(queries)diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.tsindex 4e525eefa..313d9204a 100644--- a/src/analytics/analytics.controller.ts+++ b/src/analytics/analytics.controller.ts@@ -179,12 +179,11 @@ const customLogDTO = (]}-const getElValue = el => {+export const getElValue = el => {if (el === undefined || el === null || el === 'NULL') return 'NULL'return mysql.escape(el)}-// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5const isPerformanceValid = (perf: any) => {diff --git a/src/captcha/captcha.controller.ts b/src/captcha/captcha.controller.tsindex 6e59fe584..eb08eb508 100644--- a/src/captcha/captcha.controller.ts+++ b/src/captcha/captcha.controller.ts@@ -11,6 +11,7 @@ import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { ManualDTO } from './dtos/manual.dto'import { ValidateDTO } from './dtos/validate.dto'+import { AutomaticDTO } from './dtos/automatic.dto'import { GenerateDTO, DEFAULT_THEME } from './dtos/generate.dto'dayjs.extend(utc)@@ -37,12 +38,12 @@ export class CaptchaController {this.logger.log({ generateDTO }, 'POST /captcha/generate')const {- theme = DEFAULT_THEME,+ theme = DEFAULT_THEME, pid,} = generateDTOconsole.log(generateDTO)- return await this.captchaService.generateCaptcha(theme)+ return await this.captchaService.generateCaptcha(theme, pid)}@Post('/verify-manual')@@ -52,15 +53,21 @@ export class CaptchaController {async verifyManual(@Body() manualDTO: ManualDTO,@Req() request: Request,+ @Headers() headers,@Res({ passthrough: true }) response: Response,): Promise<any> {this.logger.log({ manualDTO }, 'POST /captcha/verify-manual')+ const { 'user-agent': userAgent } = headers+ // todo: add origin checks+// @ts-ignoreconst tokenCookie = request?.cookies?.[CAPTCHA_COOKIE_KEY]- const { code, hash } = manualDTO+ const {+ code, hash, pid,+ } = manualDTO- if (!this.captchaService.verifyCaptcha(code, hash)) {+ if (!this.captchaService.verifyCaptcha(code, hash, pid)) {throw new ForbiddenException('Incorrect captcha')}@@ -86,11 +93,14 @@ export class CaptchaController {const token = this.captchaService.generateToken(TEST_PUBLIC_KEY, hash, timestamp, false)+ await this.captchaService.logCaptchaPass(pid, userAgent, headers, timestamp, true)+return {success: true,token,timestamp,hash,+ pid,}}@@ -99,10 +109,17 @@ export class CaptchaController {@UseGuards(BotDetectionGuard)@BotDetection()async autoVerifiable(+ @Body() automaticDTO: AutomaticDTO,@Req() request: Request,+ @Headers() headers,@Res({ passthrough: true }) response: Response,): Promise<any> {- this.logger.log(null, 'POST /captcha/verify')+ this.logger.log(automaticDTO, 'POST /captcha/verify')++ const { 'user-agent': userAgent } = headers+ // todo: add origin checks++ const { pid } = automaticDTO// @ts-ignorelet tokenCookie = request?.cookies?.[CAPTCHA_COOKIE_KEY]@@ -152,6 +169,8 @@ export class CaptchaController {const token = this.captchaService.generateToken(TEST_PUBLIC_KEY, null, timestamp, true)+ await this.captchaService.logCaptchaPass(pid, userAgent, headers, timestamp, false)+return {success: true,token,diff --git a/src/captcha/captcha.service.ts b/src/captcha/captcha.service.tsindex 3067e76da..6f645f922 100644--- a/src/captcha/captcha.service.ts+++ b/src/captcha/captcha.service.ts@@ -1,17 +1,27 @@import {Injectable, BadRequestException,} from '@nestjs/common'+import { hash } from 'blake3'import * as svgCaptcha from 'svg-captcha'import * as CryptoJS from 'crypto-js'-import { hash } from 'blake3'import * as _toLower from 'lodash/toLower'+import * as UAParser from 'ua-parser-js'+import * as _map from 'lodash/map'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'-import { isDevelopment } from '../common/constants'+import { AppLoggerService } from '../logger/logger.service'+import {+ isDevelopment, redis, REDIS_LOG_CAPTCHA_CACHE_KEY,+} from '../common/constants'+import { getElValue } from '../analytics/analytics.controller'import {CAPTCHA_SALT, CAPTCHA_ENCRYPTION_KEY,} from '../common/constants'+dayjs.extend(utc)+interface GeneratedCaptcha {data: stringhash: string@@ -45,11 +55,50 @@ const TEST_SECRET_KEY = 'wfOw1Jw3JAjcrHaQFvIvBS4qdG'export const CAPTCHA_COOKIE_KEY = 'swetrix-captcha-token'+const captchaDTO = (+ pid: string,+ dv: string,+ br: string,+ os: string,+ cc: string,+ isManual: boolean,+ timestamp: number+): Array<string | number> => {+ return [+ pid,+ dv,+ br,+ os,+ cc,+ isManual ? 1 : 0,+ dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),+ ]+}+@Injectable()export class CaptchaService {constructor(+ private readonly logger: AppLoggerService,) { }+ async logCaptchaPass(pid: string, userAgent: string, headers: any, timestamp: number, isManual: boolean): Promise<void> {+ const ua = UAParser(userAgent)+ const dv = ua.device.type || 'desktop'+ const br = ua.browser.name+ const os = ua.os.name+ const cc = headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry']++ const dto = captchaDTO(pid, dv, br, os, cc, isManual, timestamp)++ const values = `(${_map(dto, getElValue).join(',')})`++ try {+ await redis.rpush(REDIS_LOG_CAPTCHA_CACHE_KEY, values)+ } catch (e) {+ this.logger.error(`[CaptchaService -> logCaptchaPass] ${e}`)+ }+ }+generateToken(publicKey: string, hash: string, timestamp: number, autoVerified: boolean): string {// todo: lookup the secret key from the public key in the databaseconst secretKey = TEST_SECRET_KEY@@ -82,11 +131,11 @@ export class CaptchaService {return true}- hashCaptcha(text: string): string {+ hashCaptcha(text: string, pid: string): string {return hash(captchaString(text)).toString('hex')}- async generateCaptcha(theme: string): Promise<GeneratedCaptcha> {+ async generateCaptcha(theme: string, pid: string): Promise<GeneratedCaptcha> {const themeParams = theme === 'light' ? {} : {background: '#1f2937',color: true,@@ -99,7 +148,7 @@ export class CaptchaService {...themeParams,})const hash = this.hashCaptcha(- _toLower(captcha.text)+ _toLower(captcha.text), pid,)return {@@ -108,8 +157,8 @@ export class CaptchaService {}}- verifyCaptcha(text: string, hash: string): boolean {- return hash === this.hashCaptcha(text)+ verifyCaptcha(text: string, hash: string, pid: string): boolean {+ return hash === this.hashCaptcha(text, pid)}incrementManuallyVerified(tokenCaptcha: TokenCaptcha): TokenCaptcha {diff --git a/src/captcha/dtos/automatic.dto.ts b/src/captcha/dtos/automatic.dto.tsnew file mode 100644index 000000000..fcb6fc175--- /dev/null+++ b/src/captcha/dtos/automatic.dto.ts@@ -0,0 +1,12 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'++export class AutomaticDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'A unique project ID',+ })+ @IsNotEmpty()+ pid: string+}diff --git a/src/captcha/dtos/generate.dto.ts b/src/captcha/dtos/generate.dto.tsindex a65dcc05c..25f1c9c73 100644--- a/src/captcha/dtos/generate.dto.ts+++ b/src/captcha/dtos/generate.dto.ts@@ -1,9 +1,17 @@import { ApiProperty } from '@nestjs/swagger'-import { IsOptional } from 'class-validator'+import { IsOptional, IsNotEmpty } from 'class-validator'export const DEFAULT_THEME = 'light'export class GenerateDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'A unique project ID',+ })+ @IsNotEmpty()+ pid: string+@ApiProperty({default: DEFAULT_THEME,required: false,diff --git a/src/captcha/dtos/manual.dto.ts b/src/captcha/dtos/manual.dto.tsindex 9b32ea7bd..444cf4217 100644--- a/src/captcha/dtos/manual.dto.ts+++ b/src/captcha/dtos/manual.dto.ts@@ -2,6 +2,14 @@ import { ApiProperty } from '@nestjs/swagger'import { IsNotEmpty } from 'class-validator'export class ManualDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'A unique project ID',+ })+ @IsNotEmpty()+ pid: string+@ApiProperty({required: true,})diff --git a/src/common/constants.ts b/src/common/constants.tsindex 717792a1c..b0ff2c51e 100644--- a/src/common/constants.ts+++ b/src/common/constants.ts@@ -166,6 +166,7 @@ const getRedisProjectKey = (pid: string) => `pid_${pid}`const getRedisUserCountKey = (uid: string) => `user_c_${uid}`const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'+const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours@@ -208,6 +209,7 @@ export {redisProjectCacheTimeout,UNIQUE_SESSION_LIFE_TIME,REDIS_LOG_DATA_CACHE_KEY,+ REDIS_LOG_CAPTCHA_CACHE_KEY,GDPR_EXPORT_TIMEFRAME,getRedisUserCountKey,redisProjectCountCacheTimeout,diff --git a/src/project/project.controller.ts b/src/project/project.controller.tsindex 873ee9141..b33b40cb9 100644--- a/src/project/project.controller.ts+++ b/src/project/project.controller.ts@@ -120,8 +120,6 @@ export class ProjectController {const totalMonthlyEvents = await this.projectService.getRedisCount(userId)- console.log(where)-paginated.results = _map(paginated.results, p => ({...p,isOwner: true,