📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
fix: Files clean-up handling
Blue Mouse53459d594f6d
diff --git a/backend/apps/cloud/src/data-import/data-import.controller.ts b/backend/apps/cloud/src/data-import/data-import.controller.tsindex 519b636ad..8f4cd8d8d 100644--- a/backend/apps/cloud/src/data-import/data-import.controller.ts+++ b/backend/apps/cloud/src/data-import/data-import.controller.ts@@ -104,10 +104,13 @@ export class DataImportController {)}- const dataImport = await this.dataImportService.create(- projectId,- dto.provider,- )+ let dataImport+ try {+ dataImport = await this.dataImportService.create(projectId, dto.provider)+ } catch (error) {+ this.cleanupFile(file.path)+ throw error+ }try {await this.importQueue.add(diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.tsindex 8555dfbe5..c45fb3693 100644--- a/backend/apps/cloud/src/data-import/data-import.processor.ts+++ b/backend/apps/cloud/src/data-import/data-import.processor.ts@@ -29,102 +29,108 @@ export class DataImportProcessor extends WorkerHost {async process(job: Job<DataImportJobData>): Promise<void> {const { id, importId, projectId, provider, filePath } = job.data- this.logger.log(- `Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`,- )-- await this.dataImportService.markProcessing(id)-- const mapper = getMapper(provider)- if (!mapper) {- await this.dataImportService.markFailed(- id,- `Unsupported provider: ${provider}`,+ try {+ this.logger.log(+ `Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`,)- return- }- let importedRows = 0- let totalRows = 0- let minDate: string | null = null- let maxDate: string | null = null+ await this.dataImportService.markProcessing(id)- const analyticsBatch: Record<string, unknown>[] = []- const customEVBatch: Record<string, unknown>[] = []+ const mapper = getMapper(provider)+ if (!mapper) {+ await this.dataImportService.markFailed(+ id,+ `Unsupported provider: ${provider}`,+ )+ return+ }- try {- for await (const row of mapper.createRowStream(- filePath,- projectId,- importId,- )) {- totalRows++-- const created = row.data.created as string- if (created) {- if (!minDate || created < minDate) minDate = created- if (!maxDate || created > maxDate) maxDate = created- }+ let importedRows = 0+ let totalRows = 0+ let minDate: string | null = null+ let maxDate: string | null = null++ const analyticsBatch: Record<string, unknown>[] = []+ const customEVBatch: Record<string, unknown>[] = []- if (row.table === 'analytics') {- analyticsBatch.push(row.data)- } else {- customEVBatch.push(row.data)+ try {+ for await (const row of mapper.createRowStream(+ filePath,+ projectId,+ importId,+ )) {+ totalRows++++ const created = row.data.created as string+ if (created) {+ if (!minDate || created < minDate) minDate = created+ if (!maxDate || created > maxDate) maxDate = created+ }++ if (row.table === 'analytics') {+ analyticsBatch.push(row.data)+ } else {+ customEVBatch.push(row.data)+ }++ if (analyticsBatch.length >= BATCH_SIZE) {+ await this.flushBatch('analytics', analyticsBatch)+ importedRows += analyticsBatch.length+ analyticsBatch.length = 0+ }++ if (customEVBatch.length >= BATCH_SIZE) {+ await this.flushBatch('customEV', customEVBatch)+ importedRows += customEVBatch.length+ customEVBatch.length = 0+ }++ if (totalRows % 10000 === 0) {+ await job.updateProgress({+ importedRows:+ importedRows + analyticsBatch.length + customEVBatch.length,+ totalRows,+ })+ }}- if (analyticsBatch.length >= BATCH_SIZE) {+ if (analyticsBatch.length > 0) {await this.flushBatch('analytics', analyticsBatch)importedRows += analyticsBatch.length- analyticsBatch.length = 0}- if (customEVBatch.length >= BATCH_SIZE) {+ if (customEVBatch.length > 0) {await this.flushBatch('customEV', customEVBatch)importedRows += customEVBatch.length- customEVBatch.length = 0- }-- if (totalRows % 10000 === 0) {- await job.updateProgress({- importedRows:- importedRows + analyticsBatch.length + customEVBatch.length,- totalRows,- })}- }-- if (analyticsBatch.length > 0) {- await this.flushBatch('analytics', analyticsBatch)- importedRows += analyticsBatch.length- }-- if (customEVBatch.length > 0) {- await this.flushBatch('customEV', customEVBatch)- importedRows += customEVBatch.length- }- await this.dataImportService.markCompleted(id, {- importedRows,- invalidRows: 0,- totalRows: importedRows,- dateFrom: minDate ? new Date(minDate) : null,- dateTo: maxDate ? new Date(maxDate) : null,- })+ await this.dataImportService.markCompleted(id, {+ importedRows,+ invalidRows: 0,+ totalRows: importedRows,+ dateFrom: minDate ? new Date(minDate) : null,+ dateTo: maxDate ? new Date(maxDate) : null,+ })- this.logger.log(`Import ${id} completed: ${importedRows} rows imported`)- } catch (error) {- this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)-- // Rollback any partially inserted rows- try {- await this.dataImportService.cleanupImportedRows(projectId, importId)- } catch (cleanupError) {- this.logger.error(- `Failed to rollback partial import ${id}: ${cleanupError.message}`,+ this.logger.log(+ `Import ${id} completed: ${importedRows} rows imported`,)- }+ } catch (error) {+ this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)++ try {+ await this.dataImportService.cleanupImportedRows(+ projectId,+ importId,+ )+ } catch (cleanupError) {+ this.logger.error(+ `Failed to rollback partial import ${id}: ${cleanupError.message}`,+ )+ }- await this.dataImportService.markFailed(id, error.message)+ await this.dataImportService.markFailed(id, error.message)+ }} finally {this.cleanupFile(filePath)}diff --git a/backend/apps/community/src/data-import/data-import.controller.ts b/backend/apps/community/src/data-import/data-import.controller.tsindex 8482a735f..4ba84c384 100644--- a/backend/apps/community/src/data-import/data-import.controller.ts+++ b/backend/apps/community/src/data-import/data-import.controller.ts@@ -100,10 +100,13 @@ export class DataImportController {)}- const dataImport = await this.dataImportService.create(- projectId,- dto.provider,- )+ let dataImport+ try {+ dataImport = await this.dataImportService.create(projectId, dto.provider)+ } catch (error) {+ this.cleanupFile(file.path)+ throw error+ }try {await this.importQueue.add(diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.tsindex 63cdc97fd..d877dc031 100644--- a/backend/apps/community/src/data-import/data-import.processor.ts+++ b/backend/apps/community/src/data-import/data-import.processor.ts@@ -28,111 +28,116 @@ export class DataImportProcessor extends WorkerHost {async process(job: Job<DataImportJobData>): Promise<void> {const { importId, projectId, provider, filePath } = job.data- this.logger.log(- `Processing import ${importId} for project ${projectId} (provider: ${provider})`,- )-- await this.dataImportService.markProcessing(importId, projectId)-- const mapper = getMapper(provider)- if (!mapper) {- await this.dataImportService.markFailed(- importId,- projectId,- `Unsupported provider: ${provider}`,+ try {+ this.logger.log(+ `Processing import ${importId} for project ${projectId} (provider: ${provider})`,)- return- }- let importedRows = 0- let totalRows = 0- let minDate: string | null = null- let maxDate: string | null = null+ await this.dataImportService.markProcessing(importId, projectId)- const analyticsBatch: Record<string, unknown>[] = []- const customEVBatch: Record<string, unknown>[] = []+ const mapper = getMapper(provider)+ if (!mapper) {+ await this.dataImportService.markFailed(+ importId,+ projectId,+ `Unsupported provider: ${provider}`,+ )+ return+ }- try {- for await (const row of mapper.createRowStream(- filePath,- projectId,- importId,- )) {- totalRows++-- const created = row.data.created as string- if (created) {- if (!minDate || created < minDate) minDate = created- if (!maxDate || created > maxDate) maxDate = created- }+ let importedRows = 0+ let totalRows = 0+ let minDate: string | null = null+ let maxDate: string | null = null- if (row.table === 'analytics') {- analyticsBatch.push(row.data)- } else {- customEVBatch.push(row.data)+ const analyticsBatch: Record<string, unknown>[] = []+ const customEVBatch: Record<string, unknown>[] = []++ try {+ for await (const row of mapper.createRowStream(+ filePath,+ projectId,+ importId,+ )) {+ totalRows++++ const created = row.data.created as string+ if (created) {+ if (!minDate || created < minDate) minDate = created+ if (!maxDate || created > maxDate) maxDate = created+ }++ if (row.table === 'analytics') {+ analyticsBatch.push(row.data)+ } else {+ customEVBatch.push(row.data)+ }++ if (analyticsBatch.length >= BATCH_SIZE) {+ await this.flushBatch('analytics', analyticsBatch)+ importedRows += analyticsBatch.length+ analyticsBatch.length = 0+ }++ if (customEVBatch.length >= BATCH_SIZE) {+ await this.flushBatch('customEV', customEVBatch)+ importedRows += customEVBatch.length+ customEVBatch.length = 0+ }++ if (totalRows % 10000 === 0) {+ await job.updateProgress({+ importedRows:+ importedRows + analyticsBatch.length + customEVBatch.length,+ totalRows,+ })+ }}- if (analyticsBatch.length >= BATCH_SIZE) {+ if (analyticsBatch.length > 0) {await this.flushBatch('analytics', analyticsBatch)importedRows += analyticsBatch.length- analyticsBatch.length = 0}- if (customEVBatch.length >= BATCH_SIZE) {+ if (customEVBatch.length > 0) {await this.flushBatch('customEV', customEVBatch)importedRows += customEVBatch.length- customEVBatch.length = 0- }-- if (totalRows % 10000 === 0) {- await job.updateProgress({- importedRows:- importedRows + analyticsBatch.length + customEVBatch.length,- totalRows,- })}- }-- if (analyticsBatch.length > 0) {- await this.flushBatch('analytics', analyticsBatch)- importedRows += analyticsBatch.length- }- if (customEVBatch.length > 0) {- await this.flushBatch('customEV', customEVBatch)- importedRows += customEVBatch.length- }+ await this.dataImportService.markCompleted(importId, projectId, {+ importedRows,+ invalidRows: 0,+ totalRows: importedRows,+ dateFrom: minDate ? minDate.split(' ')[0] : null,+ dateTo: maxDate ? maxDate.split(' ')[0] : null,+ })- await this.dataImportService.markCompleted(importId, projectId, {- importedRows,- invalidRows: 0,- totalRows: importedRows,- dateFrom: minDate ? minDate.split(' ')[0] : null,- dateTo: maxDate ? maxDate.split(' ')[0] : null,- })+ this.logger.log(+ `Import ${importId} completed: ${importedRows} rows imported`,+ )+ } catch (error) {+ this.logger.error(+ `Import ${importId} failed: ${error.message}`,+ error.stack,+ )- this.logger.log(- `Import ${importId} completed: ${importedRows} rows imported`,- )- } catch (error) {- this.logger.error(- `Import ${importId} failed: ${error.message}`,- error.stack,- )+ try {+ await this.dataImportService.cleanupImportedRows(+ projectId,+ importId,+ )+ } catch (cleanupError) {+ this.logger.error(+ `Failed to rollback partial import ${importId}: ${cleanupError.message}`,+ )+ }- try {- await this.dataImportService.cleanupImportedRows(projectId, importId)- } catch (cleanupError) {- this.logger.error(- `Failed to rollback partial import ${importId}: ${cleanupError.message}`,+ await this.dataImportService.markFailed(+ importId,+ projectId,+ error.message,)}-- await this.dataImportService.markFailed(- importId,- projectId,- error.message,- )} finally {this.cleanupFile(filePath)}