Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix: Check for import status before processing

Blue Mouse4b8a43beb338
diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.ts
index d64b66a14..bb68b50fb 100644
--- a/backend/apps/cloud/src/data-import/data-import.processor.ts
+++ b/backend/apps/cloud/src/data-import/data-import.processor.ts
@@ -6,6 +6,7 @@ import { Logger } from '@nestjs/common'
import { Job } from 'bullmq'
import { DataImportService } from './data-import.service'
+import { DataImportStatus } from './entity/data-import.entity'
import { getMapper } from './mappers'
import { ImportError } from './mappers/mapper.interface'
import { clickhouse } from '../common/integrations/clickhouse'
@@ -41,6 +42,25 @@ export class DataImportProcessor extends WorkerHost {
`Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`,
)
+ const dataImport = await this.dataImportService.findById(id)
+
+ if (!dataImport) {
+ this.logger.warn(`Import record ${id} not found, skipping`)
+ return
+ }
+
+ if (dataImport.status === DataImportStatus.COMPLETED) {
+ this.logger.log(`Import ${id} already completed, skipping reprocessing`)
+ return
+ }
+
+ if (dataImport.status === DataImportStatus.PROCESSING) {
+ this.logger.warn(
+ `Import ${id} has status PROCESSING (stalled job replay), cleaning up partial data before reprocessing`,
+ )
+ await this.dataImportService.cleanupImportedRows(projectId, importId)
+ }
+
await this.dataImportService.markProcessing(id)
if (!filePath) {
diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.ts
index c5c60649f..63f60b59e 100644
--- a/backend/apps/cloud/src/data-import/data-import.service.ts
+++ b/backend/apps/cloud/src/data-import/data-import.service.ts
@@ -105,6 +105,10 @@ export class DataImportService {
})
}
+ async findById(id: number): Promise<DataImport | null> {
+ return this.dataImportRepository.findOneBy({ id })
+ }
+
async findOne(projectId: string, importId: number): Promise<DataImport> {
const dataImport = await this.dataImportRepository.findOne({
where: { importId, projectId },
diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts
index 0ede6681f..0d0646d1b 100644
--- a/backend/apps/community/src/data-import/data-import.processor.ts
+++ b/backend/apps/community/src/data-import/data-import.processor.ts
@@ -6,6 +6,7 @@ import { Logger } from '@nestjs/common'
import { Job } from 'bullmq'
import { DataImportService } from './data-import.service'
+import { DataImportStatus } from './entity/data-import.entity'
import { getMapper } from './mappers'
import { clickhouse } from '../common/integrations/clickhouse'
@@ -39,6 +40,25 @@ export class DataImportProcessor extends WorkerHost {
`Processing import ${importId} for project ${projectId} (provider: ${provider})`,
)
+ const dataImport = await this.dataImportService.findOne(
+ projectId,
+ importId,
+ )
+
+ if (dataImport.status === DataImportStatus.COMPLETED) {
+ this.logger.log(
+ `Import ${importId} already completed, skipping reprocessing`,
+ )
+ return
+ }
+
+ if (dataImport.status === DataImportStatus.PROCESSING) {
+ this.logger.warn(
+ `Import ${importId} has status PROCESSING (stalled job replay), cleaning up partial data before reprocessing`,
+ )
+ await this.dataImportService.cleanupImportedRows(projectId, importId)
+ }
+
await this.dataImportService.markProcessing(importId, projectId)
if (!filePath) {