Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Various reliability improvements to workers

Edward Cui5e29bef5ab4a
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 1ab97e079..924134ec5 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -88,25 +88,46 @@ export async function registerCsvParseWorker() {
await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([job]) => {
const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
+ let stream: any = null;
+ let processingTimeout: NodeJS.Timeout | null = null;
+
try {
const quotaTracker = await ImportLimiter.createQuotaTracker(organization);
const chunkSize = 5000;
+ const MAX_ROWS = 10_000_000; // 10 million rows max
+ const PROCESSING_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes max
+
let chunk: UmamiEvent[] = [];
let totalAccepted = 0;
let totalSkippedQuota = 0;
let totalSkippedDate = 0;
+ let totalRowsProcessed = 0;
let chunksSent = 0;
- const stream = isR2Storage
+ stream = isR2Storage
? await createR2FileStream(storageLocation, source)
: await createLocalFileStream(storageLocation, source);
await ImportStatusManager.updateStatus(importId, "processing");
+ // Set timeout to prevent indefinite processing
+ processingTimeout = setTimeout(() => {
+ if (stream) {
+ stream.destroy(new Error("Import processing timeout exceeded"));
+ }
+ }, PROCESSING_TIMEOUT_MS);
+
const isDateInRange = createDateRangeFilter(startDate, endDate);
for await (const data of stream) {
+ totalRowsProcessed++;
+
+ // Enforce row count limit to prevent memory exhaustion
+ if (totalRowsProcessed > MAX_ROWS) {
+ throw new Error(`Import exceeds maximum row limit of ${MAX_ROWS.toLocaleString()}`);
+ }
+
// Skip rows with missing or invalid dates
if (!data.created_at) {
continue;
@@ -185,14 +206,32 @@ export async function registerCsvParseWorker() {
allChunksSent: true,
});
} catch (error) {
- console.error("Error in CSV parse worker:", error);
- await ImportStatusManager.updateStatus(
- importId,
- "failed",
- error instanceof Error ? error.message : "Unknown error occurred"
- );
- throw error;
+ console.error(`[Import ${importId}] Error in CSV parse worker:`, error);
+
+ // Sanitize error message to avoid exposing internal details
+ const safeMessage = error instanceof Error
+ ? error.message.replace(/\/[^\s]+/g, '[path]').substring(0, 500)
+ : "An unexpected error occurred during import processing";
+
+ await ImportStatusManager.updateStatus(importId, "failed", safeMessage);
+
+ // Don't re-throw - worker should continue processing other jobs
+ console.error(`[Import ${importId}] Import failed, worker continuing`);
} finally {
+ // Clean up timeout
+ if (processingTimeout) {
+ clearTimeout(processingTimeout);
+ }
+
+ // Ensure stream is destroyed to prevent memory leaks
+ if (stream && typeof stream.destroy === "function") {
+ try {
+ stream.destroy();
+ } catch (streamError) {
+ console.warn(`[Import ${importId}] Failed to destroy stream:`, streamError);
+ }
+ }
+
// Clean up file - don't throw on failure to prevent worker crashes
const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
if (!deleteResult.success) {
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 3ec14d2ca..9e0c9d20f 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -27,8 +27,14 @@ export async function registerDataInsertWorker() {
return;
} catch (error) {
console.error(`[Import ${importId}] Failed to mark as completed:`, error);
- await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
- throw error;
+ // Try to update to failed status, but don't crash worker
+ try {
+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
+ } catch (updateError) {
+ console.error(`[Import ${importId}] Could not update status to failed:`, updateError);
+ }
+ // Don't re-throw - worker should continue
+ return;
}
}
@@ -58,12 +64,20 @@ export async function registerDataInsertWorker() {
console.log(`[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`);
} catch (error) {
console.error(`[Import ${importId}] ClickHouse insert failed:`, error);
- await ImportStatusManager.updateStatus(
- importId,
- "failed",
- `Data insertion failed: ${error instanceof Error ? error.message : "Unknown error"}`
- );
- throw error;
+
+ // Sanitize error message
+ const safeMessage = error instanceof Error
+ ? `Data insertion failed: ${error.message.substring(0, 500)}`
+ : "Data insertion failed due to unknown error";
+
+ try {
+ await ImportStatusManager.updateStatus(importId, "failed", safeMessage);
+ } catch (updateError) {
+ console.error(`[Import ${importId}] Could not update status to failed:`, updateError);
+ }
+
+ // Don't re-throw - worker should continue processing other jobs
+ console.error(`[Import ${importId}] Import chunk failed, worker continuing`);
}
});
}