📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
fix: one active import guard is not atomic
Blue Mouse6356e4152a81
diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.tsindex c45fb3693..bce677e2a 100644--- a/backend/apps/cloud/src/data-import/data-import.processor.ts+++ b/backend/apps/cloud/src/data-import/data-import.processor.ts@@ -112,17 +112,12 @@ export class DataImportProcessor extends WorkerHost {dateTo: maxDate ? new Date(maxDate) : null,})- this.logger.log(- `Import ${id} completed: ${importedRows} rows imported`,- )+ this.logger.log(`Import ${id} completed: ${importedRows} rows imported`)} catch (error) {this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)try {- await this.dataImportService.cleanupImportedRows(- projectId,- importId,- )+ await this.dataImportService.cleanupImportedRows(projectId, importId)} catch (cleanupError) {this.logger.error(`Failed to rollback partial import ${id}: ${cleanupError.message}`,diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.tsindex 65da68a73..37f0355e9 100644--- a/backend/apps/cloud/src/data-import/data-import.service.ts+++ b/backend/apps/cloud/src/data-import/data-import.service.ts@@ -5,7 +5,7 @@ import {Logger,} from '@nestjs/common'import { InjectRepository } from '@nestjs/typeorm'-import { Repository } from 'typeorm'+import { DataSource, Repository } from 'typeorm'import {DataImport,@@ -13,6 +13,7 @@ import {MAX_IMPORT_ID,} from './entity/data-import.entity'import { clickhouse } from '../common/integrations/clickhouse'+import { Project } from '../project/entity/project.entity'const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'@@ -23,10 +24,14 @@ export class DataImportService {constructor(@InjectRepository(DataImport)private readonly dataImportRepository: Repository<DataImport>,+ private readonly dataSource: DataSource,) {}- private async getNextImportId(projectId: string): Promise<number> {- const existingImportIds = await this.dataImportRepository+ private async getNextImportId(+ repository: Repository<DataImport>,+ projectId: string,+ ): Promise<number> {+ const existingImportIds = await repository.createQueryBuilder('di').select('di.importId', 'importId').where('di.projectId = :projectId', { projectId })@@ -60,29 +65,37 @@ export class DataImportService {}async create(projectId: string, provider: string): Promise<DataImport> {- const active = await this.dataImportRepository.findOne({- where: [- { projectId, status: DataImportStatus.PENDING },- { projectId, status: DataImportStatus.PROCESSING },- ],- })+ return this.dataSource.transaction(async (manager) => {+ await manager.getRepository(Project).findOneOrFail({+ where: { id: projectId },+ lock: { mode: 'pessimistic_write' },+ })- if (active) {- throw new BadRequestException(- 'An import is already in progress for this project. Please wait for it to complete before starting a new one.',- )- }+ const repository = manager.getRepository(DataImport)+ const active = await repository.findOne({+ where: [+ { projectId, status: DataImportStatus.PENDING },+ { projectId, status: DataImportStatus.PROCESSING },+ ],+ })- const importId = await this.getNextImportId(projectId)+ if (active) {+ throw new BadRequestException(+ 'An import is already in progress for this project. Please wait for it to complete before starting a new one.',+ )+ }- const dataImport = this.dataImportRepository.create({- projectId,- importId,- provider,- status: DataImportStatus.PENDING,- })+ const importId = await this.getNextImportId(repository, projectId)- return this.dataImportRepository.save(dataImport)+ const dataImport = repository.create({+ projectId,+ importId,+ provider,+ status: DataImportStatus.PENDING,+ })++ return repository.save(dataImport)+ })}async findAllByProject(projectId: string): Promise<DataImport[]> {diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.tsindex d877dc031..9055442f6 100644--- a/backend/apps/community/src/data-import/data-import.processor.ts+++ b/backend/apps/community/src/data-import/data-import.processor.ts@@ -122,10 +122,7 @@ export class DataImportProcessor extends WorkerHost {)try {- await this.dataImportService.cleanupImportedRows(- projectId,- importId,- )+ await this.dataImportService.cleanupImportedRows(projectId, importId)} catch (cleanupError) {this.logger.error(`Failed to rollback partial import ${importId}: ${cleanupError.message}`,diff --git a/backend/apps/community/src/data-import/data-import.service.ts b/backend/apps/community/src/data-import/data-import.service.tsindex f7f4bd12d..ac9be0c50 100644--- a/backend/apps/community/src/data-import/data-import.service.ts+++ b/backend/apps/community/src/data-import/data-import.service.ts@@ -4,6 +4,7 @@ import {NotFoundException,Logger,} from '@nestjs/common'+import { randomUUID } from 'crypto'import {DataImport,@@ -11,13 +12,64 @@ import {MAX_IMPORT_ID,} from './entity/data-import.entity'import { clickhouse } from '../common/integrations/clickhouse'+import { redis } from '../common/constants'const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'+const IMPORT_IN_PROGRESS_ERROR =+ 'An import is already in progress for this project. Please wait for it to complete before starting a new one.'+const IMPORT_CREATE_LOCK_TTL_MS = 10_000+const IMPORT_CREATE_LOCK_WAIT_MS = 5_000+const IMPORT_CREATE_LOCK_RETRY_MS = 100@Injectable()export class DataImportService {private readonly logger = new Logger(DataImportService.name)+ private getCreateLockKey(projectId: string): string {+ return `data-import:create:${projectId}`+ }++ private async acquireCreateLock(projectId: string): Promise<string> {+ const token = randomUUID()+ const lockKey = this.getCreateLockKey(projectId)+ const deadline = Date.now() + IMPORT_CREATE_LOCK_WAIT_MS++ while (Date.now() < deadline) {+ const acquired = await redis.set(+ lockKey,+ token,+ 'PX',+ IMPORT_CREATE_LOCK_TTL_MS,+ 'NX',+ )++ if (acquired === 'OK') {+ return token+ }++ await new Promise((resolve) =>+ setTimeout(resolve, IMPORT_CREATE_LOCK_RETRY_MS),+ )+ }++ throw new BadRequestException(IMPORT_IN_PROGRESS_ERROR)+ }++ private async releaseCreateLock(+ projectId: string,+ token: string,+ ): Promise<void> {+ await redis.eval(+ `if redis.call("GET", KEYS[1]) == ARGV[1] then+ return redis.call("DEL", KEYS[1])+ end+ return 0`,+ 1,+ this.getCreateLockKey(projectId),+ token,+ )+ }+private async getNextId(projectId: string): Promise<number> {const result = await clickhouse.query({@@ -53,57 +105,61 @@ export class DataImportService {}async create(projectId: string, provider: string): Promise<DataImport> {- const active = await this.findActiveByProject(projectId)- if (active) {- throw new BadRequestException(- 'An import is already in progress for this project. Please wait for it to complete before starting a new one.',- )- }-- const id = await this.getNextId(projectId)+ const lockToken = await this.acquireCreateLock(projectId)- const now = new Date()- .toISOString()- .replace('T', ' ')- .replace('Z', '')- .slice(0, 19)+ try {+ const active = await this.findActiveByProject(projectId)+ if (active) {+ throw new BadRequestException(IMPORT_IN_PROGRESS_ERROR)+ }- await clickhouse.insert({- table: `${CLICKHOUSE_DB}.data_import`,- values: [- {- id,- projectId,- provider,- status: 'pending',- dateFrom: null,- dateTo: null,- totalRows: 0,- importedRows: 0,- invalidRows: 0,- errorMessage: null,- createdAt: now,- finishedAt: null,- version: 1,- },- ],- format: 'JSONEachRow',- })+ const id = await this.getNextId(projectId)++ const now = new Date()+ .toISOString()+ .replace('T', ' ')+ .replace('Z', '')+ .slice(0, 19)++ await clickhouse.insert({+ table: `${CLICKHOUSE_DB}.data_import`,+ values: [+ {+ id,+ projectId,+ provider,+ status: 'pending',+ dateFrom: null,+ dateTo: null,+ totalRows: 0,+ importedRows: 0,+ invalidRows: 0,+ errorMessage: null,+ createdAt: now,+ finishedAt: null,+ version: 1,+ },+ ],+ format: 'JSONEachRow',+ })- return {- id,- projectId,- provider,- status: DataImportStatus.PENDING,- dateFrom: null,- dateTo: null,- totalRows: 0,- importedRows: 0,- invalidRows: 0,- errorMessage: null,- createdAt: now,- finishedAt: null,- version: 1,+ return {+ id,+ projectId,+ provider,+ status: DataImportStatus.PENDING,+ dateFrom: null,+ dateTo: null,+ totalRows: 0,+ importedRows: 0,+ invalidRows: 0,+ errorMessage: null,+ createdAt: now,+ finishedAt: null,+ version: 1,+ }+ } finally {+ await this.releaseCreateLock(projectId, lockToken)}}