Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Error handling

Edward Cui335699826992
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.ts
index fcdcce2b0..52d5d9319 100644
--- a/server/src/workers/importInitiation.ts
+++ b/server/src/workers/importInitiation.ts
@@ -8,129 +8,145 @@ import { clickhouse } from "../db/clickhouse/clickhouse.js";
import { processResults } from "../api/analytics/utils.js";
await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {
- const { tempFilePath, site, importId, source } = job[0].data;
-
- const query = await clickhouse.query({
- query: `
- SELECT count() AS count
- FROM events
- WHERE site_id = {siteId:UInt16}
- AND import_id IS NOT NULL
- `,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- });
- const result = await processResults<{ count: number }>(query);
- const importedRowCount = result[0].count;
-
- const maxImportedRows = 1000000;
- const remainingRows = maxImportedRows - importedRowCount;
- if (remainingRows <= 0) {
- console.log(`πŸ›‘ No remaining rows can be imported (already imported: ${importedRowCount}).`);
- await fs.promises.unlink(tempFilePath);
- return;
- }
+ try {
+ const { tempFilePath, site, importId, source } = job[0].data;
+
+ const query = await clickhouse.query({
+ query: `
+ SELECT count() AS count
+ FROM events
+ WHERE site_id = {siteId:UInt16}
+ AND import_id IS NOT NULL
+ `,
+ format: "JSONEachRow",
+ query_params: {
+ siteId: Number(site),
+ },
+ });
+ const result = await processResults<{ count: number }>(query);
+ const importedRowCount = result[0].count;
- const chunkSize = 1000;
- const maxChunks = Math.ceil(remainingRows / chunkSize);
- let chunkNumber = 0;
- let chunk: any[] = [];
- let shouldStop = false;
+ const maxImportedRows = 1000000;
+ const remainingRows = maxImportedRows - importedRowCount;
+ if (remainingRows <= 0) {
+ console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);
+ await fs.promises.unlink(tempFilePath);
+ return;
+ }
- const stream = fs.createReadStream(tempFilePath);
+ const chunkSize = 1000;
+ const maxChunks = Math.ceil(remainingRows / chunkSize);
+ let chunkNumber = 0;
+ let chunk: any[] = [];
+ let shouldStop = false;
- const arraysAreEqual = (arr1: string[], arr2: string[]) => {
- if (arr1.length !== arr2.length) {
- return false;
- }
- return arr1.every((value, index) => value === arr2[index]);
- }
+ const stream = fs.createReadStream(tempFilePath);
- const validateHeaders = (headers: string[]): boolean => {
- switch (source) {
- case "umami":
- return arraysAreEqual(umamiHeaders, headers);
- default:
+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {
+ if (arr1.length !== arr2.length) {
return false;
+ }
+ return arr1.every((value, index) => value === arr2[index]);
}
- };
- const cleanup = async (reason: string) => {
- console.log(`πŸ›‘ Stopping CSV processing: ${reason}`);
- shouldStop = true;
-
- if (!stream.destroyed) {
- stream.destroy();
- }
+ const validateHeaders = (headers: string[]): boolean => {
+ switch (source) {
+ case "umami":
+ return arraysAreEqual(umamiHeaders, headers);
+ default:
+ return false;
+ }
+ };
- try {
- await fs.promises.unlink(tempFilePath);
- console.log(`πŸ—‘οΈ Deleted temporary file: ${tempFilePath}`);
- } catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
- }
- };
+ const cleanup = async (reason: string) => {
+ console.log(`Stopping CSV processing: ${reason}`);
+ shouldStop = true;
- parseStream(stream, { headers: true })
- .on("headers", async (headers) => {
- if (!validateHeaders(headers)) {
- await cleanup("Header validation failed");
- return;
- }
- })
- .on("data", async (row) => {
- if (shouldStop) {
- return;
+ if (!stream.destroyed) {
+ stream.destroy();
}
- chunk.push(row);
-
- if (chunk.length >= chunkSize) {
- chunkNumber++;
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
+ }
+ };
- if (chunkNumber > maxChunks) {
- await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);
+ parseStream(stream, {headers: true})
+ .on("headers", async (headers) => {
+ if (!validateHeaders(headers)) {
+ await cleanup("Header validation failed");
+ return;
+ }
+ })
+ .on("data", async (row) => {
+ if (shouldStop) {
return;
}
- await boss.send(processImportChunkQueue, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- });
- chunk = [];
- }
- })
- .on("end", async (rowCount: number) => {
- if (!shouldStop && chunk.length > 0) {
- chunkNumber++;
-
- await boss.send(processImportChunkQueue, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- });
- }
-
- if (!shouldStop) {
- console.log(`βœ… Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
-
+ chunk.push(row);
+
+ if (chunk.length >= chunkSize) {
+ chunkNumber++;
+
+ if (chunkNumber > maxChunks) {
+ await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);
+ return;
+ }
+
+ try {
+ await boss.send(processImportChunkQueue, {
+ site,
+ importId,
+ source,
+ chunk,
+ chunkNumber,
+ });
+ } catch (error) {
+ console.error("Failed to send chunk to queue:", error);
+ await cleanup("Error sending chunk to processing queue");
+ }
+
+ chunk = [];
+ }
+ })
+ .on("end", async (rowCount: number) => {
try {
- await fs.promises.unlink(tempFilePath);
- console.log(`πŸ—‘οΈ Deleted temporary file: ${tempFilePath}`);
+ if (!shouldStop && chunk.length > 0) {
+ chunkNumber++;
+
+ await boss.send(processImportChunkQueue, {
+ site,
+ importId,
+ source,
+ chunk,
+ chunkNumber,
+ });
+ }
+
+ if (!shouldStop) {
+ console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
+
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
+ }
+ }
} catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
+ console.error("Error during final CSV chunk processing:", error);
+ await cleanup("Error during final chunk processing");
}
- }
- })
- .on("error", async (error) => {
- console.error("CSV parsing error:", error);
- await cleanup("CSV parsing error occurred");
- });
+ })
+ .on("error", async (error) => {
+ console.error("CSV parsing error:", error);
+ await cleanup("CSV parsing error occurred");
+ });
+ } catch (error) {
+ console.error("Error in importInitiationQueue worker:", error);
+ // optional: notify, requeue, or cleanup
+ }
});