Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

some improvements

Blue Mousefd1849c23478
diff --git a/migrations/clickhouse/2023_03_11.js b/migrations/clickhouse/2023_03_11.js
new file mode 100644
index 000000000..a0999c7dd
--- /dev/null
+++ b/migrations/clickhouse/2023_03_11.js
@@ -0,0 +1,19 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ `CREATE TABLE IF NOT EXISTS ${dbName}.captcha
+ (
+ pid FixedString(12),
+ dv Nullable(String),
+ br Nullable(String),
+ os Nullable(String),
+ cc Nullable(FixedString(2)),
+ manuallyPassed UInt8,
+ created DateTime
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+]
+
+queriesRunner(queries)
diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.ts
index 4e525eefa..313d9204a 100644
--- a/src/analytics/analytics.controller.ts
+++ b/src/analytics/analytics.controller.ts
@@ -179,12 +179,11 @@ const customLogDTO = (
]
}
-const getElValue = el => {
+export const getElValue = el => {
if (el === undefined || el === null || el === 'NULL') return 'NULL'
return mysql.escape(el)
}
-
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
diff --git a/src/captcha/captcha.controller.ts b/src/captcha/captcha.controller.ts
index 6e59fe584..eb08eb508 100644
--- a/src/captcha/captcha.controller.ts
+++ b/src/captcha/captcha.controller.ts
@@ -11,6 +11,7 @@ import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { ManualDTO } from './dtos/manual.dto'
import { ValidateDTO } from './dtos/validate.dto'
+import { AutomaticDTO } from './dtos/automatic.dto'
import { GenerateDTO, DEFAULT_THEME } from './dtos/generate.dto'
dayjs.extend(utc)
@@ -37,12 +38,12 @@ export class CaptchaController {
this.logger.log({ generateDTO }, 'POST /captcha/generate')
const {
- theme = DEFAULT_THEME,
+ theme = DEFAULT_THEME, pid,
} = generateDTO
console.log(generateDTO)
- return await this.captchaService.generateCaptcha(theme)
+ return await this.captchaService.generateCaptcha(theme, pid)
}
@Post('/verify-manual')
@@ -52,15 +53,21 @@ export class CaptchaController {
async verifyManual(
@Body() manualDTO: ManualDTO,
@Req() request: Request,
+ @Headers() headers,
@Res({ passthrough: true }) response: Response,
): Promise<any> {
this.logger.log({ manualDTO }, 'POST /captcha/verify-manual')
+ const { 'user-agent': userAgent } = headers
+ // todo: add origin checks
+
// @ts-ignore
const tokenCookie = request?.cookies?.[CAPTCHA_COOKIE_KEY]
- const { code, hash } = manualDTO
+ const {
+ code, hash, pid,
+ } = manualDTO
- if (!this.captchaService.verifyCaptcha(code, hash)) {
+ if (!this.captchaService.verifyCaptcha(code, hash, pid)) {
throw new ForbiddenException('Incorrect captcha')
}
@@ -86,11 +93,14 @@ export class CaptchaController {
const token = this.captchaService.generateToken(TEST_PUBLIC_KEY, hash, timestamp, false)
+ await this.captchaService.logCaptchaPass(pid, userAgent, headers, timestamp, true)
+
return {
success: true,
token,
timestamp,
hash,
+ pid,
}
}
@@ -99,10 +109,17 @@ export class CaptchaController {
@UseGuards(BotDetectionGuard)
@BotDetection()
async autoVerifiable(
+ @Body() automaticDTO: AutomaticDTO,
@Req() request: Request,
+ @Headers() headers,
@Res({ passthrough: true }) response: Response,
): Promise<any> {
- this.logger.log(null, 'POST /captcha/verify')
+ this.logger.log(automaticDTO, 'POST /captcha/verify')
+
+ const { 'user-agent': userAgent } = headers
+ // todo: add origin checks
+
+ const { pid } = automaticDTO
// @ts-ignore
let tokenCookie = request?.cookies?.[CAPTCHA_COOKIE_KEY]
@@ -152,6 +169,8 @@ export class CaptchaController {
const token = this.captchaService.generateToken(TEST_PUBLIC_KEY, null, timestamp, true)
+ await this.captchaService.logCaptchaPass(pid, userAgent, headers, timestamp, false)
+
return {
success: true,
token,
diff --git a/src/captcha/captcha.service.ts b/src/captcha/captcha.service.ts
index 3067e76da..6f645f922 100644
--- a/src/captcha/captcha.service.ts
+++ b/src/captcha/captcha.service.ts
@@ -1,17 +1,27 @@
import {
Injectable, BadRequestException,
} from '@nestjs/common'
+import { hash } from 'blake3'
import * as svgCaptcha from 'svg-captcha'
import * as CryptoJS from 'crypto-js'
-import { hash } from 'blake3'
import * as _toLower from 'lodash/toLower'
+import * as UAParser from 'ua-parser-js'
+import * as _map from 'lodash/map'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
-import { isDevelopment } from '../common/constants'
+import { AppLoggerService } from '../logger/logger.service'
+import {
+ isDevelopment, redis, REDIS_LOG_CAPTCHA_CACHE_KEY,
+} from '../common/constants'
+import { getElValue } from '../analytics/analytics.controller'
import {
CAPTCHA_SALT, CAPTCHA_ENCRYPTION_KEY,
} from '../common/constants'
+dayjs.extend(utc)
+
interface GeneratedCaptcha {
data: string
hash: string
@@ -45,11 +55,50 @@ const TEST_SECRET_KEY = 'wfOw1Jw3JAjcrHaQFvIvBS4qdG'
export const CAPTCHA_COOKIE_KEY = 'swetrix-captcha-token'
+const captchaDTO = (
+ pid: string,
+ dv: string,
+ br: string,
+ os: string,
+ cc: string,
+ isManual: boolean,
+ timestamp: number
+): Array<string | number> => {
+ return [
+ pid,
+ dv,
+ br,
+ os,
+ cc,
+ isManual ? 1 : 0,
+ dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),
+ ]
+}
+
@Injectable()
export class CaptchaService {
constructor(
+ private readonly logger: AppLoggerService,
) { }
+ async logCaptchaPass(pid: string, userAgent: string, headers: any, timestamp: number, isManual: boolean): Promise<void> {
+ const ua = UAParser(userAgent)
+ const dv = ua.device.type || 'desktop'
+ const br = ua.browser.name
+ const os = ua.os.name
+ const cc = headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry']
+
+ const dto = captchaDTO(pid, dv, br, os, cc, isManual, timestamp)
+
+ const values = `(${_map(dto, getElValue).join(',')})`
+
+ try {
+ await redis.rpush(REDIS_LOG_CAPTCHA_CACHE_KEY, values)
+ } catch (e) {
+ this.logger.error(`[CaptchaService -> logCaptchaPass] ${e}`)
+ }
+ }
+
generateToken(publicKey: string, hash: string, timestamp: number, autoVerified: boolean): string {
// todo: lookup the secret key from the public key in the database
const secretKey = TEST_SECRET_KEY
@@ -82,11 +131,11 @@ export class CaptchaService {
return true
}
- hashCaptcha(text: string): string {
+ hashCaptcha(text: string, pid: string): string {
return hash(captchaString(text)).toString('hex')
}
- async generateCaptcha(theme: string): Promise<GeneratedCaptcha> {
+ async generateCaptcha(theme: string, pid: string): Promise<GeneratedCaptcha> {
const themeParams = theme === 'light' ? {} : {
background: '#1f2937',
color: true,
@@ -99,7 +148,7 @@ export class CaptchaService {
...themeParams,
})
const hash = this.hashCaptcha(
- _toLower(captcha.text)
+ _toLower(captcha.text), pid,
)
return {
@@ -108,8 +157,8 @@ export class CaptchaService {
}
}
- verifyCaptcha(text: string, hash: string): boolean {
- return hash === this.hashCaptcha(text)
+ verifyCaptcha(text: string, hash: string, pid: string): boolean {
+ return hash === this.hashCaptcha(text, pid)
}
incrementManuallyVerified(tokenCaptcha: TokenCaptcha): TokenCaptcha {
diff --git a/src/captcha/dtos/automatic.dto.ts b/src/captcha/dtos/automatic.dto.ts
new file mode 100644
index 000000000..fcb6fc175
--- /dev/null
+++ b/src/captcha/dtos/automatic.dto.ts
@@ -0,0 +1,12 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+
+export class AutomaticDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'A unique project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+}
diff --git a/src/captcha/dtos/generate.dto.ts b/src/captcha/dtos/generate.dto.ts
index a65dcc05c..25f1c9c73 100644
--- a/src/captcha/dtos/generate.dto.ts
+++ b/src/captcha/dtos/generate.dto.ts
@@ -1,9 +1,17 @@
import { ApiProperty } from '@nestjs/swagger'
-import { IsOptional } from 'class-validator'
+import { IsOptional, IsNotEmpty } from 'class-validator'
export const DEFAULT_THEME = 'light'
export class GenerateDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'A unique project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+
@ApiProperty({
default: DEFAULT_THEME,
required: false,
diff --git a/src/captcha/dtos/manual.dto.ts b/src/captcha/dtos/manual.dto.ts
index 9b32ea7bd..444cf4217 100644
--- a/src/captcha/dtos/manual.dto.ts
+++ b/src/captcha/dtos/manual.dto.ts
@@ -2,6 +2,14 @@ import { ApiProperty } from '@nestjs/swagger'
import { IsNotEmpty } from 'class-validator'
export class ManualDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'A unique project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+
@ApiProperty({
required: true,
})
diff --git a/src/common/constants.ts b/src/common/constants.ts
index 717792a1c..b0ff2c51e 100644
--- a/src/common/constants.ts
+++ b/src/common/constants.ts
@@ -166,6 +166,7 @@ const getRedisProjectKey = (pid: string) => `pid_${pid}`
const getRedisUserCountKey = (uid: string) => `user_c_${uid}`
const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'
+const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'
const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'
const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'
const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
@@ -208,6 +209,7 @@ export {
redisProjectCacheTimeout,
UNIQUE_SESSION_LIFE_TIME,
REDIS_LOG_DATA_CACHE_KEY,
+ REDIS_LOG_CAPTCHA_CACHE_KEY,
GDPR_EXPORT_TIMEFRAME,
getRedisUserCountKey,
redisProjectCountCacheTimeout,
diff --git a/src/project/project.controller.ts b/src/project/project.controller.ts
index 873ee9141..b33b40cb9 100644
--- a/src/project/project.controller.ts
+++ b/src/project/project.controller.ts
@@ -120,8 +120,6 @@ export class ProjectController {
const totalMonthlyEvents = await this.projectService.getRedisCount(userId)
- console.log(where)
-
paginated.results = _map(paginated.results, p => ({
...p,
isOwner: true,