Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix: one active import guard is not atomic

Blue Mouse6356e4152a81
diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.ts
index c45fb3693..bce677e2a 100644
--- a/backend/apps/cloud/src/data-import/data-import.processor.ts
+++ b/backend/apps/cloud/src/data-import/data-import.processor.ts
@@ -112,17 +112,12 @@ export class DataImportProcessor extends WorkerHost {
dateTo: maxDate ? new Date(maxDate) : null,
})
- this.logger.log(
- `Import ${id} completed: ${importedRows} rows imported`,
- )
+ this.logger.log(`Import ${id} completed: ${importedRows} rows imported`)
} catch (error) {
this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)
try {
- await this.dataImportService.cleanupImportedRows(
- projectId,
- importId,
- )
+ await this.dataImportService.cleanupImportedRows(projectId, importId)
} catch (cleanupError) {
this.logger.error(
`Failed to rollback partial import ${id}: ${cleanupError.message}`,
diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.ts
index 65da68a73..37f0355e9 100644
--- a/backend/apps/cloud/src/data-import/data-import.service.ts
+++ b/backend/apps/cloud/src/data-import/data-import.service.ts
@@ -5,7 +5,7 @@ import {
Logger,
} from '@nestjs/common'
import { InjectRepository } from '@nestjs/typeorm'
-import { Repository } from 'typeorm'
+import { DataSource, Repository } from 'typeorm'
import {
DataImport,
@@ -13,6 +13,7 @@ import {
MAX_IMPORT_ID,
} from './entity/data-import.entity'
import { clickhouse } from '../common/integrations/clickhouse'
+import { Project } from '../project/entity/project.entity'
const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'
@@ -23,10 +24,14 @@ export class DataImportService {
constructor(
@InjectRepository(DataImport)
private readonly dataImportRepository: Repository<DataImport>,
+ private readonly dataSource: DataSource,
) {}
- private async getNextImportId(projectId: string): Promise<number> {
- const existingImportIds = await this.dataImportRepository
+ private async getNextImportId(
+ repository: Repository<DataImport>,
+ projectId: string,
+ ): Promise<number> {
+ const existingImportIds = await repository
.createQueryBuilder('di')
.select('di.importId', 'importId')
.where('di.projectId = :projectId', { projectId })
@@ -60,29 +65,37 @@ export class DataImportService {
}
async create(projectId: string, provider: string): Promise<DataImport> {
- const active = await this.dataImportRepository.findOne({
- where: [
- { projectId, status: DataImportStatus.PENDING },
- { projectId, status: DataImportStatus.PROCESSING },
- ],
- })
+ return this.dataSource.transaction(async (manager) => {
+ await manager.getRepository(Project).findOneOrFail({
+ where: { id: projectId },
+ lock: { mode: 'pessimistic_write' },
+ })
- if (active) {
- throw new BadRequestException(
- 'An import is already in progress for this project. Please wait for it to complete before starting a new one.',
- )
- }
+ const repository = manager.getRepository(DataImport)
+ const active = await repository.findOne({
+ where: [
+ { projectId, status: DataImportStatus.PENDING },
+ { projectId, status: DataImportStatus.PROCESSING },
+ ],
+ })
- const importId = await this.getNextImportId(projectId)
+ if (active) {
+ throw new BadRequestException(
+ 'An import is already in progress for this project. Please wait for it to complete before starting a new one.',
+ )
+ }
- const dataImport = this.dataImportRepository.create({
- projectId,
- importId,
- provider,
- status: DataImportStatus.PENDING,
- })
+ const importId = await this.getNextImportId(repository, projectId)
- return this.dataImportRepository.save(dataImport)
+ const dataImport = repository.create({
+ projectId,
+ importId,
+ provider,
+ status: DataImportStatus.PENDING,
+ })
+
+ return repository.save(dataImport)
+ })
}
async findAllByProject(projectId: string): Promise<DataImport[]> {
diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts
index d877dc031..9055442f6 100644
--- a/backend/apps/community/src/data-import/data-import.processor.ts
+++ b/backend/apps/community/src/data-import/data-import.processor.ts
@@ -122,10 +122,7 @@ export class DataImportProcessor extends WorkerHost {
)
try {
- await this.dataImportService.cleanupImportedRows(
- projectId,
- importId,
- )
+ await this.dataImportService.cleanupImportedRows(projectId, importId)
} catch (cleanupError) {
this.logger.error(
`Failed to rollback partial import ${importId}: ${cleanupError.message}`,
diff --git a/backend/apps/community/src/data-import/data-import.service.ts b/backend/apps/community/src/data-import/data-import.service.ts
index f7f4bd12d..ac9be0c50 100644
--- a/backend/apps/community/src/data-import/data-import.service.ts
+++ b/backend/apps/community/src/data-import/data-import.service.ts
@@ -4,6 +4,7 @@ import {
NotFoundException,
Logger,
} from '@nestjs/common'
+import { randomUUID } from 'crypto'
import {
DataImport,
@@ -11,13 +12,64 @@ import {
MAX_IMPORT_ID,
} from './entity/data-import.entity'
import { clickhouse } from '../common/integrations/clickhouse'
+import { redis } from '../common/constants'
const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'
+const IMPORT_IN_PROGRESS_ERROR =
+ 'An import is already in progress for this project. Please wait for it to complete before starting a new one.'
+const IMPORT_CREATE_LOCK_TTL_MS = 10_000
+const IMPORT_CREATE_LOCK_WAIT_MS = 5_000
+const IMPORT_CREATE_LOCK_RETRY_MS = 100
@Injectable()
export class DataImportService {
private readonly logger = new Logger(DataImportService.name)
+ private getCreateLockKey(projectId: string): string {
+ return `data-import:create:${projectId}`
+ }
+
+ private async acquireCreateLock(projectId: string): Promise<string> {
+ const token = randomUUID()
+ const lockKey = this.getCreateLockKey(projectId)
+ const deadline = Date.now() + IMPORT_CREATE_LOCK_WAIT_MS
+
+ while (Date.now() < deadline) {
+ const acquired = await redis.set(
+ lockKey,
+ token,
+ 'PX',
+ IMPORT_CREATE_LOCK_TTL_MS,
+ 'NX',
+ )
+
+ if (acquired === 'OK') {
+ return token
+ }
+
+ await new Promise((resolve) =>
+ setTimeout(resolve, IMPORT_CREATE_LOCK_RETRY_MS),
+ )
+ }
+
+ throw new BadRequestException(IMPORT_IN_PROGRESS_ERROR)
+ }
+
+ private async releaseCreateLock(
+ projectId: string,
+ token: string,
+ ): Promise<void> {
+ await redis.eval(
+ `if redis.call("GET", KEYS[1]) == ARGV[1] then
+ return redis.call("DEL", KEYS[1])
+ end
+ return 0`,
+ 1,
+ this.getCreateLockKey(projectId),
+ token,
+ )
+ }
+
private async getNextId(projectId: string): Promise<number> {
const result = await clickhouse
.query({
@@ -53,57 +105,61 @@ export class DataImportService {
}
async create(projectId: string, provider: string): Promise<DataImport> {
- const active = await this.findActiveByProject(projectId)
- if (active) {
- throw new BadRequestException(
- 'An import is already in progress for this project. Please wait for it to complete before starting a new one.',
- )
- }
-
- const id = await this.getNextId(projectId)
+ const lockToken = await this.acquireCreateLock(projectId)
- const now = new Date()
- .toISOString()
- .replace('T', ' ')
- .replace('Z', '')
- .slice(0, 19)
+ try {
+ const active = await this.findActiveByProject(projectId)
+ if (active) {
+ throw new BadRequestException(IMPORT_IN_PROGRESS_ERROR)
+ }
- await clickhouse.insert({
- table: `${CLICKHOUSE_DB}.data_import`,
- values: [
- {
- id,
- projectId,
- provider,
- status: 'pending',
- dateFrom: null,
- dateTo: null,
- totalRows: 0,
- importedRows: 0,
- invalidRows: 0,
- errorMessage: null,
- createdAt: now,
- finishedAt: null,
- version: 1,
- },
- ],
- format: 'JSONEachRow',
- })
+ const id = await this.getNextId(projectId)
+
+ const now = new Date()
+ .toISOString()
+ .replace('T', ' ')
+ .replace('Z', '')
+ .slice(0, 19)
+
+ await clickhouse.insert({
+ table: `${CLICKHOUSE_DB}.data_import`,
+ values: [
+ {
+ id,
+ projectId,
+ provider,
+ status: 'pending',
+ dateFrom: null,
+ dateTo: null,
+ totalRows: 0,
+ importedRows: 0,
+ invalidRows: 0,
+ errorMessage: null,
+ createdAt: now,
+ finishedAt: null,
+ version: 1,
+ },
+ ],
+ format: 'JSONEachRow',
+ })
- return {
- id,
- projectId,
- provider,
- status: DataImportStatus.PENDING,
- dateFrom: null,
- dateTo: null,
- totalRows: 0,
- importedRows: 0,
- invalidRows: 0,
- errorMessage: null,
- createdAt: now,
- finishedAt: null,
- version: 1,
+ return {
+ id,
+ projectId,
+ provider,
+ status: DataImportStatus.PENDING,
+ dateFrom: null,
+ dateTo: null,
+ totalRows: 0,
+ importedRows: 0,
+ invalidRows: 0,
+ errorMessage: null,
+ createdAt: now,
+ finishedAt: null,
+ version: 1,
+ }
+ } finally {
+ await this.releaseCreateLock(projectId, lockToken)
}
}