🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify error handling
Edward Cui6be21b577523
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex cd7d7ddaa..d0e8fcfad 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -22,6 +22,14 @@ const getImportDataHeaders = (platform: string) => {}};+const safeUpdateStatusToFailed = async (importId: string, errorMessage: string) => {+ try {+ await updateImportStatus(importId, "failed", errorMessage);+ } catch (updateError) {+ logger.error({ importId, error: updateError }, "Could not update status to failed");+ }+};+const createR2FileStream = async (storageLocation: string, platform: string) => {logger.info({ storageLocation }, "Reading from R2");const fileStream = await r2Storage.getImportFileStream(storageLocation);@@ -89,7 +97,7 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {const quotaTracker = await ImportQuotaTracker.create(organization);const chunkSize = 5000;- const INACTIVITY_TIMEOUT_MS = 90 * 1000; // 90 seconds of inactivity+ const INACTIVITY_TIMEOUT_MS = 90 * 1000;let chunk: UmamiEvent[] = [];let totalSkippedQuota = 0;@@ -99,16 +107,13 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {? await createR2FileStream(storageLocation, platform): await createLocalFileStream(storageLocation, platform);- // Add explicit error handler before starting to consume the streamstream.on("error", error => {logger.error({ importId, error }, "Stream error");- // Error will be caught by the outer try/catch});await updateImportStatus(importId, "processing");logger.info({ importId, platform, organization }, "Started processing CSV import");- // Helper to reset inactivity timeout - resets whenever progress is madeconst resetTimeout = () => {if (processingTimeout) {clearTimeout(processingTimeout);@@ -142,7 +147,6 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {continue;}- // Event passed all filters - add to chunkchunk.push(data);totalProcessed++;@@ -155,12 +159,10 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {allChunksSent: false,});chunk = [];- // Reset timeout after successful chunk sendresetTimeout();}}- // Send final chunk if any data remainsif (chunk.length > 0) {await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {site,@@ -169,11 +171,9 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {chunk,allChunksSent: false,});- // Reset timeout after final chunk sendresetTimeout();}- // Check if some events couldn't be imported due to quotasif (totalSkippedQuota > 0) {const quotaSummary = quotaTracker.getSummary();const errorMessage =@@ -188,7 +188,6 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {return;}- // Send finalization signal with total chunk countlogger.info({ importId, totalProcessed, totalSkippedQuota }, "CSV parsing completed successfully");await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {site,@@ -199,18 +198,13 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {});} catch (error) {logger.error({ importId, error }, "Error in CSV parse worker");-- await updateImportStatus(importId, "failed", "An unexpected error occurred during import processing");-- // Don't re-throw - worker should continue processing other jobs+ await safeUpdateStatusToFailed(importId, "An unexpected error occurred during import processing");logger.error({ importId }, "Import failed, worker continuing");} finally {- // Clean up timeoutif (processingTimeout) {clearTimeout(processingTimeout);}- // Ensure stream is destroyed to prevent memory leaksif (stream) {try {stream.destroy();@@ -219,12 +213,16 @@ export async function createCsvParseWorker(jobQueue: IJobQueue) {}}- // Clean up file - don't throw on failure to prevent worker crashes- const deleteResult = await deleteImportFile(storageLocation, isR2Storage);- if (!deleteResult.success) {- logger.warn({ importId, error: deleteResult.error }, "File cleanup failed, will remain in storage");- // File will be orphaned but import status is already recorded- // importCleanupService.ts handles orphans+ // Clean up file - log errors but don't throw to prevent worker crashes+ try {+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);+ if (!deleteResult.success) {+ logger.warn({ importId, error: deleteResult.error }, "File cleanup failed, will remain in storage");+ // File will be orphaned but import status is already recorded+ // importCleanupService.ts handles orphans+ }+ } catch (deleteError) {+ logger.error({ importId, error: deleteError }, "Critical error during file cleanup");}}});diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex ef311f1a6..cdab8a131 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -16,33 +16,28 @@ const getImportDataMapping = (platform: string) => {}};+const safeUpdateStatusToFailed = async (importId: string, errorMessage: string) => {+ try {+ await updateImportStatus(importId, "failed", errorMessage);+ } catch (updateError) {+ logger.error({ importId, error: updateError }, "Could not update status to failed");+ }+};+export async function createDataInsertWorker(jobQueue: IJobQueue) {await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {const { site, importId, platform, chunk, allChunksSent } = job;- if (allChunksSent) {- try {+ try {+ if (allChunksSent) {await updateImportStatus(importId, "completed");logger.info({ importId }, "Import completed successfully");return;- } catch (error) {- logger.error({ importId, error }, "Failed to mark as completed");- // Try to update to failed status, but don't crash worker- try {- await updateImportStatus(importId, "failed", "Failed to complete import");- } catch (updateError) {- logger.error({ importId, error: updateError }, "Could not update status to failed");- }- // Don't re-throw - worker should continue- return;}- }- try {const dataMapper = getImportDataMapping(platform);const transformedRecords = dataMapper.transform(chunk, site, importId);- // Insert to ClickHouse (critical - must succeed)await clickhouse.insert({table: "events",values: transformedRecords,@@ -62,16 +57,12 @@ export async function createDataInsertWorker(jobQueue: IJobQueue) {// Don't throw - data is safely in ClickHouse, progress can be off slightly}} catch (error) {- logger.error({ importId, error }, "ClickHouse insert failed");+ logger.error({ importId, error }, "Worker processing failed");- try {- await updateImportStatus(importId, "failed", "Data insertion failed due to unknown error");- } catch (updateError) {- logger.error({ importId, error: updateError }, "Could not update status to failed");- }+ const errorMessage = allChunksSent ? "Failed to complete import" : "Data insertion failed due to unknown error";+ await safeUpdateStatusToFailed(importId, errorMessage);- // Don't re-throw - worker should continue processing other jobs- logger.error({ importId }, "Import chunk failed, worker continuing");+ logger.error({ importId }, "Import operation failed, worker continuing");}});}