πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Error handling
Edward Cui335699826992
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.tsindex fcdcce2b0..52d5d9319 100644--- a/server/src/workers/importInitiation.ts+++ b/server/src/workers/importInitiation.ts@@ -8,129 +8,145 @@ import { clickhouse } from "../db/clickhouse/clickhouse.js";import { processResults } from "../api/analytics/utils.js";await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {- const { tempFilePath, site, importId, source } = job[0].data;-- const query = await clickhouse.query({- query: `- SELECT count() AS count- FROM events- WHERE site_id = {siteId:UInt16}- AND import_id IS NOT NULL- `,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- });- const result = await processResults<{ count: number }>(query);- const importedRowCount = result[0].count;-- const maxImportedRows = 1000000;- const remainingRows = maxImportedRows - importedRowCount;- if (remainingRows <= 0) {- console.log(`π No remaining rows can be imported (already imported: ${importedRowCount}).`);- await fs.promises.unlink(tempFilePath);- return;- }+ try {+ const { tempFilePath, site, importId, source } = job[0].data;++ const query = await clickhouse.query({+ query: `+ SELECT count() AS count+ FROM events+ WHERE site_id = {siteId:UInt16}+ AND import_id IS NOT NULL+ `,+ format: "JSONEachRow",+ query_params: {+ siteId: Number(site),+ },+ });+ const result = await processResults<{ count: number }>(query);+ const importedRowCount = result[0].count;- const chunkSize = 1000;- const maxChunks = Math.ceil(remainingRows / chunkSize);- let chunkNumber = 0;- let chunk: any[] = [];- let shouldStop = false;+ const maxImportedRows = 1000000;+ const remainingRows = maxImportedRows - importedRowCount;+ if (remainingRows <= 0) {+ console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);+ await fs.promises.unlink(tempFilePath);+ return;+ }- const stream = fs.createReadStream(tempFilePath);+ const chunkSize = 1000;+ const maxChunks = Math.ceil(remainingRows / chunkSize);+ let chunkNumber = 0;+ let chunk: any[] = [];+ let shouldStop = false;- const arraysAreEqual = (arr1: string[], arr2: string[]) => {- if (arr1.length !== arr2.length) {- return false;- }- return arr1.every((value, index) => value === arr2[index]);- }+ const stream = fs.createReadStream(tempFilePath);- const validateHeaders = (headers: string[]): boolean => {- switch (source) {- case "umami":- return arraysAreEqual(umamiHeaders, headers);- default:+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {+ if (arr1.length !== arr2.length) {return false;+ }+ return arr1.every((value, index) => value === arr2[index]);}- };- const cleanup = async (reason: string) => {- console.log(`π Stopping CSV processing: ${reason}`);- shouldStop = true;-- if (!stream.destroyed) {- stream.destroy();- }+ const validateHeaders = (headers: string[]): boolean => {+ switch (source) {+ case "umami":+ return arraysAreEqual(umamiHeaders, headers);+ default:+ return false;+ }+ };- try {- await fs.promises.unlink(tempFilePath);- console.log(`ποΈ Deleted temporary file: ${tempFilePath}`);- } catch (error) {- console.error(`Failed to delete file ${tempFilePath}:`, error);- }- };+ const cleanup = async (reason: string) => {+ console.log(`Stopping CSV processing: ${reason}`);+ shouldStop = true;- parseStream(stream, { headers: true })- .on("headers", async (headers) => {- if (!validateHeaders(headers)) {- await cleanup("Header validation failed");- return;- }- })- .on("data", async (row) => {- if (shouldStop) {- return;+ if (!stream.destroyed) {+ stream.destroy();}- chunk.push(row);-- if (chunk.length >= chunkSize) {- chunkNumber++;+ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);+ }+ };- if (chunkNumber > maxChunks) {- await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);+ parseStream(stream, {headers: true})+ .on("headers", async (headers) => {+ if (!validateHeaders(headers)) {+ await cleanup("Header validation failed");+ return;+ }+ })+ .on("data", async (row) => {+ if (shouldStop) {return;}- await boss.send(processImportChunkQueue, {- site,- importId,- source,- chunk,- chunkNumber,- });- chunk = [];- }- })- .on("end", async (rowCount: number) => {- if (!shouldStop && chunk.length > 0) {- chunkNumber++;-- await boss.send(processImportChunkQueue, {- site,- importId,- source,- chunk,- chunkNumber,- });- }-- if (!shouldStop) {- console.log(`β Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);-+ chunk.push(row);++ if (chunk.length >= chunkSize) {+ chunkNumber++;++ if (chunkNumber > maxChunks) {+ await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);+ return;+ }++ try {+ await boss.send(processImportChunkQueue, {+ site,+ importId,+ source,+ chunk,+ chunkNumber,+ });+ } catch (error) {+ console.error("Failed to send chunk to queue:", error);+ await cleanup("Error sending chunk to processing queue");+ }++ chunk = [];+ }+ })+ .on("end", async (rowCount: number) => {try {- await fs.promises.unlink(tempFilePath);- console.log(`ποΈ Deleted temporary file: ${tempFilePath}`);+ if (!shouldStop && chunk.length > 0) {+ chunkNumber++;++ await boss.send(processImportChunkQueue, {+ site,+ importId,+ source,+ chunk,+ chunkNumber,+ });+ }++ if (!shouldStop) {+ console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);++ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);+ }+ }} catch (error) {- console.error(`Failed to delete file ${tempFilePath}:`, error);+ console.error("Error during final CSV chunk processing:", error);+ await cleanup("Error during final chunk processing");}- }- })- .on("error", async (error) => {- console.error("CSV parsing error:", error);- await cleanup("CSV parsing error occurred");- });+ })+ .on("error", async (error) => {+ console.error("CSV parsing error:", error);+ await cleanup("CSV parsing error occurred");+ });+ } catch (error) {+ console.error("Error in importInitiationQueue worker:", error);+ // optional: notify, requeue, or cleanup+ }});