πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Various improvements
Edward Cuif27382c5af51
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.tsindex 352cc149f..cdac32c98 100644--- a/server/src/workers/importInitiation.ts+++ b/server/src/workers/importInitiation.ts@@ -3,44 +3,108 @@ import { parseStream } from "@fast-csv/parse";import boss from "../lib/boss.js";import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";import { Job } from "pg-boss";+import { umamiHeaders } from "./mappings/umami.js";await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {const { tempFilePath, site, importId, source } = job[0].data;const chunkSize = 1000;+ const maxChunks = 1000; // 1 million rows limitlet chunkNumber = 0;let chunk: any[] = [];+ let shouldStop = false;const stream = fs.createReadStream(tempFilePath);+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {+ if (arr1.length !== arr2.length) {+ return false;+ }+ return arr1.every((value, index) => value === arr2[index]);+ }++ const validateHeaders = (headers: string[]): boolean => {+ switch (source) {+ case "umami":+ return arraysAreEqual(umamiHeaders, headers);+ default:+ return false;+ }+ };++ const cleanup = async (reason: string) => {+ console.log(`π Stopping CSV processing: ${reason}`);+ shouldStop = true;++ if (!stream.destroyed) {+ stream.destroy();+ }++ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`ποΈ Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);+ }+ };+parseStream(stream, { headers: true })- .on("headers", headers => console.log(headers)) // validate headers+ .on("headers", async (headers) => {+ if (!validateHeaders(headers)) {+ await cleanup("Header validation failed");+ return;+ }+ }).on("data", async (row) => {+ if (shouldStop) {+ return;+ }+chunk.push(row);if (chunk.length >= chunkSize) {+ chunkNumber++;++ if (chunkNumber > maxChunks) {+ await cleanup(`Maximum chunks limit reached (${maxChunks} chunks / 1 million rows)`);+ return;+ }+await boss.send(processImportChunkQueue, {+ site,importId,source,- site,- chunkNumber: ++chunkNumber,- records: chunk,+ chunk,+ chunkNumber,});chunk = [];}}).on("end", async (rowCount: number) => {- if (chunk.length > 0) {+ if (!shouldStop && chunk.length > 0) {+ chunkNumber++;+await boss.send(processImportChunkQueue, {+ site,importId,source,- site,- chunkNumber: ++chunkNumber,- records: chunk,+ chunk,+ chunkNumber,});}- console.log(`β Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);- })- .on("error", error => console.error("CSV parsing error:", error));- await fs.promises.unlink(tempFilePath);+ if (!shouldStop) {+ console.log(`β Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);++ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`ποΈ Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);+ }+ }+ })+ .on("error", async (error) => {+ console.error("CSV parsing error:", error);+ await cleanup("CSV parsing error occurred");+ });});diff --git a/server/src/workers/mappings/umami.ts b/server/src/workers/mappings/umami.tsindex d55d4a0c9..f939141d8 100644--- a/server/src/workers/mappings/umami.ts+++ b/server/src/workers/mappings/umami.ts@@ -1,4 +1,4 @@-import { ImportMapping } from "../../types/import.js";+import { ImportMapper } from "../../types/import.js";interface UmamiEvent {website_id: string; // Ignore@@ -43,7 +43,46 @@ interface UmamiEvent {job_id: string | null; // Ignore}-export class UmamiImportMapping implements ImportMapping<UmamiEvent> {+export const umamiHeaders = [+ "website_id",+ "session_id",+ "visit_id",+ "event_id",+ "hostname",+ "browser",+ "os",+ "device",+ "screen",+ "language",+ "country",+ "region",+ "city",+ "url_path",+ "url_query",+ "utm_source",+ "utm_medium",+ "utm_campaign",+ "utm_content",+ "utm_term",+ "referrer_path",+ "referrer_query",+ "referrer_domain",+ "page_title",+ "gclid",+ "fbclid",+ "msclkid",+ "ttclid",+ "li_fat_id",+ "twclid",+ "event_type",+ "event_name",+ "tag",+ "distinct_id",+ "created_at",+ "job_id"+];++export class UmamiImportMapper implements ImportMapper<UmamiEvent> {transform(row: UmamiEvent, headers: string[]): any {const [screenWidth, screenHeight] = row.screen?.split("x") || [null, null];