Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix: Files clean-up handling

Blue Mouse53459d594f6d
diff --git a/backend/apps/cloud/src/data-import/data-import.controller.ts b/backend/apps/cloud/src/data-import/data-import.controller.ts
index 519b636ad..8f4cd8d8d 100644
--- a/backend/apps/cloud/src/data-import/data-import.controller.ts
+++ b/backend/apps/cloud/src/data-import/data-import.controller.ts
@@ -104,10 +104,13 @@ export class DataImportController {
)
}
- const dataImport = await this.dataImportService.create(
- projectId,
- dto.provider,
- )
+ let dataImport
+ try {
+ dataImport = await this.dataImportService.create(projectId, dto.provider)
+ } catch (error) {
+ this.cleanupFile(file.path)
+ throw error
+ }
try {
await this.importQueue.add(
diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.ts
index 8555dfbe5..c45fb3693 100644
--- a/backend/apps/cloud/src/data-import/data-import.processor.ts
+++ b/backend/apps/cloud/src/data-import/data-import.processor.ts
@@ -29,102 +29,108 @@ export class DataImportProcessor extends WorkerHost {
async process(job: Job<DataImportJobData>): Promise<void> {
const { id, importId, projectId, provider, filePath } = job.data
- this.logger.log(
- `Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`,
- )
-
- await this.dataImportService.markProcessing(id)
-
- const mapper = getMapper(provider)
- if (!mapper) {
- await this.dataImportService.markFailed(
- id,
- `Unsupported provider: ${provider}`,
+ try {
+ this.logger.log(
+ `Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`,
)
- return
- }
- let importedRows = 0
- let totalRows = 0
- let minDate: string | null = null
- let maxDate: string | null = null
+ await this.dataImportService.markProcessing(id)
- const analyticsBatch: Record<string, unknown>[] = []
- const customEVBatch: Record<string, unknown>[] = []
+ const mapper = getMapper(provider)
+ if (!mapper) {
+ await this.dataImportService.markFailed(
+ id,
+ `Unsupported provider: ${provider}`,
+ )
+ return
+ }
- try {
- for await (const row of mapper.createRowStream(
- filePath,
- projectId,
- importId,
- )) {
- totalRows++
-
- const created = row.data.created as string
- if (created) {
- if (!minDate || created < minDate) minDate = created
- if (!maxDate || created > maxDate) maxDate = created
- }
+ let importedRows = 0
+ let totalRows = 0
+ let minDate: string | null = null
+ let maxDate: string | null = null
+
+ const analyticsBatch: Record<string, unknown>[] = []
+ const customEVBatch: Record<string, unknown>[] = []
- if (row.table === 'analytics') {
- analyticsBatch.push(row.data)
- } else {
- customEVBatch.push(row.data)
+ try {
+ for await (const row of mapper.createRowStream(
+ filePath,
+ projectId,
+ importId,
+ )) {
+ totalRows++
+
+ const created = row.data.created as string
+ if (created) {
+ if (!minDate || created < minDate) minDate = created
+ if (!maxDate || created > maxDate) maxDate = created
+ }
+
+ if (row.table === 'analytics') {
+ analyticsBatch.push(row.data)
+ } else {
+ customEVBatch.push(row.data)
+ }
+
+ if (analyticsBatch.length >= BATCH_SIZE) {
+ await this.flushBatch('analytics', analyticsBatch)
+ importedRows += analyticsBatch.length
+ analyticsBatch.length = 0
+ }
+
+ if (customEVBatch.length >= BATCH_SIZE) {
+ await this.flushBatch('customEV', customEVBatch)
+ importedRows += customEVBatch.length
+ customEVBatch.length = 0
+ }
+
+ if (totalRows % 10000 === 0) {
+ await job.updateProgress({
+ importedRows:
+ importedRows + analyticsBatch.length + customEVBatch.length,
+ totalRows,
+ })
+ }
}
- if (analyticsBatch.length >= BATCH_SIZE) {
+ if (analyticsBatch.length > 0) {
await this.flushBatch('analytics', analyticsBatch)
importedRows += analyticsBatch.length
- analyticsBatch.length = 0
}
- if (customEVBatch.length >= BATCH_SIZE) {
+ if (customEVBatch.length > 0) {
await this.flushBatch('customEV', customEVBatch)
importedRows += customEVBatch.length
- customEVBatch.length = 0
- }
-
- if (totalRows % 10000 === 0) {
- await job.updateProgress({
- importedRows:
- importedRows + analyticsBatch.length + customEVBatch.length,
- totalRows,
- })
}
- }
-
- if (analyticsBatch.length > 0) {
- await this.flushBatch('analytics', analyticsBatch)
- importedRows += analyticsBatch.length
- }
-
- if (customEVBatch.length > 0) {
- await this.flushBatch('customEV', customEVBatch)
- importedRows += customEVBatch.length
- }
- await this.dataImportService.markCompleted(id, {
- importedRows,
- invalidRows: 0,
- totalRows: importedRows,
- dateFrom: minDate ? new Date(minDate) : null,
- dateTo: maxDate ? new Date(maxDate) : null,
- })
+ await this.dataImportService.markCompleted(id, {
+ importedRows,
+ invalidRows: 0,
+ totalRows: importedRows,
+ dateFrom: minDate ? new Date(minDate) : null,
+ dateTo: maxDate ? new Date(maxDate) : null,
+ })
- this.logger.log(`Import ${id} completed: ${importedRows} rows imported`)
- } catch (error) {
- this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)
-
- // Rollback any partially inserted rows
- try {
- await this.dataImportService.cleanupImportedRows(projectId, importId)
- } catch (cleanupError) {
- this.logger.error(
- `Failed to rollback partial import ${id}: ${cleanupError.message}`,
+ this.logger.log(
+ `Import ${id} completed: ${importedRows} rows imported`,
)
- }
+ } catch (error) {
+ this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)
+
+ try {
+ await this.dataImportService.cleanupImportedRows(
+ projectId,
+ importId,
+ )
+ } catch (cleanupError) {
+ this.logger.error(
+ `Failed to rollback partial import ${id}: ${cleanupError.message}`,
+ )
+ }
- await this.dataImportService.markFailed(id, error.message)
+ await this.dataImportService.markFailed(id, error.message)
+ }
} finally {
this.cleanupFile(filePath)
}
diff --git a/backend/apps/community/src/data-import/data-import.controller.ts b/backend/apps/community/src/data-import/data-import.controller.ts
index 8482a735f..4ba84c384 100644
--- a/backend/apps/community/src/data-import/data-import.controller.ts
+++ b/backend/apps/community/src/data-import/data-import.controller.ts
@@ -100,10 +100,13 @@ export class DataImportController {
)
}
- const dataImport = await this.dataImportService.create(
- projectId,
- dto.provider,
- )
+ let dataImport
+ try {
+ dataImport = await this.dataImportService.create(projectId, dto.provider)
+ } catch (error) {
+ this.cleanupFile(file.path)
+ throw error
+ }
try {
await this.importQueue.add(
diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts
index 63cdc97fd..d877dc031 100644
--- a/backend/apps/community/src/data-import/data-import.processor.ts
+++ b/backend/apps/community/src/data-import/data-import.processor.ts
@@ -28,111 +28,116 @@ export class DataImportProcessor extends WorkerHost {
async process(job: Job<DataImportJobData>): Promise<void> {
const { importId, projectId, provider, filePath } = job.data
- this.logger.log(
- `Processing import ${importId} for project ${projectId} (provider: ${provider})`,
- )
-
- await this.dataImportService.markProcessing(importId, projectId)
-
- const mapper = getMapper(provider)
- if (!mapper) {
- await this.dataImportService.markFailed(
- importId,
- projectId,
- `Unsupported provider: ${provider}`,
+ try {
+ this.logger.log(
+ `Processing import ${importId} for project ${projectId} (provider: ${provider})`,
)
- return
- }
- let importedRows = 0
- let totalRows = 0
- let minDate: string | null = null
- let maxDate: string | null = null
+ await this.dataImportService.markProcessing(importId, projectId)
- const analyticsBatch: Record<string, unknown>[] = []
- const customEVBatch: Record<string, unknown>[] = []
+ const mapper = getMapper(provider)
+ if (!mapper) {
+ await this.dataImportService.markFailed(
+ importId,
+ projectId,
+ `Unsupported provider: ${provider}`,
+ )
+ return
+ }
- try {
- for await (const row of mapper.createRowStream(
- filePath,
- projectId,
- importId,
- )) {
- totalRows++
-
- const created = row.data.created as string
- if (created) {
- if (!minDate || created < minDate) minDate = created
- if (!maxDate || created > maxDate) maxDate = created
- }
+ let importedRows = 0
+ let totalRows = 0
+ let minDate: string | null = null
+ let maxDate: string | null = null
- if (row.table === 'analytics') {
- analyticsBatch.push(row.data)
- } else {
- customEVBatch.push(row.data)
+ const analyticsBatch: Record<string, unknown>[] = []
+ const customEVBatch: Record<string, unknown>[] = []
+
+ try {
+ for await (const row of mapper.createRowStream(
+ filePath,
+ projectId,
+ importId,
+ )) {
+ totalRows++
+
+ const created = row.data.created as string
+ if (created) {
+ if (!minDate || created < minDate) minDate = created
+ if (!maxDate || created > maxDate) maxDate = created
+ }
+
+ if (row.table === 'analytics') {
+ analyticsBatch.push(row.data)
+ } else {
+ customEVBatch.push(row.data)
+ }
+
+ if (analyticsBatch.length >= BATCH_SIZE) {
+ await this.flushBatch('analytics', analyticsBatch)
+ importedRows += analyticsBatch.length
+ analyticsBatch.length = 0
+ }
+
+ if (customEVBatch.length >= BATCH_SIZE) {
+ await this.flushBatch('customEV', customEVBatch)
+ importedRows += customEVBatch.length
+ customEVBatch.length = 0
+ }
+
+ if (totalRows % 10000 === 0) {
+ await job.updateProgress({
+ importedRows:
+ importedRows + analyticsBatch.length + customEVBatch.length,
+ totalRows,
+ })
+ }
}
- if (analyticsBatch.length >= BATCH_SIZE) {
+ if (analyticsBatch.length > 0) {
await this.flushBatch('analytics', analyticsBatch)
importedRows += analyticsBatch.length
- analyticsBatch.length = 0
}
- if (customEVBatch.length >= BATCH_SIZE) {
+ if (customEVBatch.length > 0) {
await this.flushBatch('customEV', customEVBatch)
importedRows += customEVBatch.length
- customEVBatch.length = 0
- }
-
- if (totalRows % 10000 === 0) {
- await job.updateProgress({
- importedRows:
- importedRows + analyticsBatch.length + customEVBatch.length,
- totalRows,
- })
}
- }
-
- if (analyticsBatch.length > 0) {
- await this.flushBatch('analytics', analyticsBatch)
- importedRows += analyticsBatch.length
- }
- if (customEVBatch.length > 0) {
- await this.flushBatch('customEV', customEVBatch)
- importedRows += customEVBatch.length
- }
+ await this.dataImportService.markCompleted(importId, projectId, {
+ importedRows,
+ invalidRows: 0,
+ totalRows: importedRows,
+ dateFrom: minDate ? minDate.split(' ')[0] : null,
+ dateTo: maxDate ? maxDate.split(' ')[0] : null,
+ })
- await this.dataImportService.markCompleted(importId, projectId, {
- importedRows,
- invalidRows: 0,
- totalRows: importedRows,
- dateFrom: minDate ? minDate.split(' ')[0] : null,
- dateTo: maxDate ? maxDate.split(' ')[0] : null,
- })
+ this.logger.log(
+ `Import ${importId} completed: ${importedRows} rows imported`,
+ )
+ } catch (error) {
+ this.logger.error(
+ `Import ${importId} failed: ${error.message}`,
+ error.stack,
+ )
- this.logger.log(
- `Import ${importId} completed: ${importedRows} rows imported`,
- )
- } catch (error) {
- this.logger.error(
- `Import ${importId} failed: ${error.message}`,
- error.stack,
- )
+ try {
+ await this.dataImportService.cleanupImportedRows(
+ projectId,
+ importId,
+ )
+ } catch (cleanupError) {
+ this.logger.error(
+ `Failed to rollback partial import ${importId}: ${cleanupError.message}`,
+ )
+ }
- try {
- await this.dataImportService.cleanupImportedRows(projectId, importId)
- } catch (cleanupError) {
- this.logger.error(
- `Failed to rollback partial import ${importId}: ${cleanupError.message}`,
+ await this.dataImportService.markFailed(
+ importId,
+ projectId,
+ error.message,
)
}
-
- await this.dataImportService.markFailed(
- importId,
- projectId,
- error.message,
- )
} finally {
this.cleanupFile(filePath)
}